diff --git a/package.json b/package.json index 63f174d..22fc8a1 100644 --- a/package.json +++ b/package.json @@ -14,7 +14,7 @@ "build:server": "tsc -p tsconfig.server.json", "start": "node dist/server/server/index.js", "gate:peer-topology-loopback": "tsx scripts/peer-assisted-benchmark.ts", - "gate:sfu-admission": "vitest run tests/sfu-resource-admission.test.ts tests/sfu-room-control.test.ts tests/media-route-controller.test.ts tests/livekit-token.test.ts tests/sfu-client.test.ts tests/route-transition.test.ts tests/peer-assisted-benchmark.test.ts tests/server-config.test.ts tests/server-http.test.ts tests/server-signal.test.ts", + "gate:sfu-admission": "vitest run tests/room-route-controller.test.ts tests/hybrid-media-router.test.ts tests/sfu-resource-admission.test.ts tests/sfu-room-control.test.ts tests/livekit-token.test.ts tests/sfu-client.test.ts tests/route-transition.test.ts tests/server-config.test.ts tests/server-http.test.ts tests/server-signal.test.ts", "gate:access-privacy": "npm run build:client && vitest run tests/server-signal.test.ts && tsx scripts/access-privacy-gate.ts", "gate:turn-udp-allocation": "tsx scripts/turn-allocation-gate.ts", "probe:native-one-viewer": "tsx scripts/native-one-viewer-gate.ts", diff --git a/src/client/media/decoded-frame-stall.ts b/src/client/media/decoded-frame-stall.ts new file mode 100644 index 0000000..64c701b --- /dev/null +++ b/src/client/media/decoded-frame-stall.ts @@ -0,0 +1,49 @@ +export const ROUTE_DECODED_FRAME_STALL_MS = 15_000; + +export class DecodedFrameStallDetector { + private identity: string | null = null; + private lastDecodedAt = 0; + private reported = false; + private paused = false; + + setPaused(paused: boolean, nowMs = Date.now()): void { + if (this.paused === paused) return; + this.paused = paused; + this.lastDecodedAt = nowMs; + this.reported = false; + } + + observe( + identity: string, + framesDecodedDelta: number | null, + nowMs = Date.now(), + ): boolean { + if (identity !== this.identity) { + this.identity = identity; + this.lastDecodedAt = nowMs; + this.reported = false; + return false; + } + if (this.paused) { + this.lastDecodedAt = nowMs; + this.reported = false; + return false; + } + if ((framesDecodedDelta ?? 0) > 0) { + this.lastDecodedAt = nowMs; + this.reported = false; + return false; + } + if (this.reported || nowMs - this.lastDecodedAt < ROUTE_DECODED_FRAME_STALL_MS) { + return false; + } + this.reported = true; + return true; + } + + reset(): void { + this.identity = null; + this.lastDecodedAt = 0; + this.reported = false; + } +} diff --git a/src/client/media/host-provisional-child.ts b/src/client/media/host-provisional-child.ts index 57d36af..668e30b 100644 --- a/src/client/media/host-provisional-child.ts +++ b/src/client/media/host-provisional-child.ts @@ -1,7 +1,7 @@ import type { IceConfig, ParticipantRouteAssignment, - ServerMessage, + PreparedRouteCandidate, SignalPayload, } from "../../shared/protocol"; import { countEndpointMediaCopies } from "../../shared/media-copy-accounting"; @@ -9,11 +9,6 @@ import { HostPeer } from "../webrtc/host-peer"; import type { PeerSnapshot } from "../types"; import type { QualityProfile } from "./quality"; -type SelectedEdgeTurn = Extract< - ServerMessage, - { type: "selected-edge-turn"; edgeKind: "peer-selected" } ->; - interface HostProvisionalInput { revision: number; assignment: ParticipantRouteAssignment; @@ -22,6 +17,7 @@ interface HostProvisionalInput { } interface HostProvisionalPrepareInput extends HostProvisionalInput { + candidate: PreparedRouteCandidate; iceConfig: IceConfig; stream: MediaStream; profile: QualityProfile; @@ -40,7 +36,7 @@ export type HostPreparedChildActivation = interface PreparedHostChild { revision: number; - peerId: string; + candidate: PreparedRouteCandidate; peer: HostPeer; failed: boolean; replacesConnectionId: string | null; @@ -49,6 +45,7 @@ interface PreparedHostChild { export class HostProvisionalChild { private prepared: PreparedHostChild | null = null; private preparedRevision: number | null = null; + private preparedCandidate: PreparedRouteCandidate | null = null; private plannedChildPeerIds: string[] = []; private promotedPeer: HostPeer | null = null; private signalingPeer: HostPeer | null = null; @@ -63,75 +60,45 @@ export class HostProvisionalChild { } if ( this.preparedRevision === input.revision && + sameCandidate(this.preparedCandidate, input.candidate) && samePeerIds(this.plannedChildPeerIds, planned) ) { return this.prepared ? !this.prepared.failed : true; } - const peerId = planned.find( - (candidate) => !input.activeChildPeerIds.includes(candidate), - ); this.discard(); this.preparedRevision = input.revision; + this.preparedCandidate = { ...input.candidate }; this.plannedChildPeerIds = planned; - if (!peerId) { - return true; - } - if (this.events.activeConnectionId?.(peerId)) { + if ( + this.events.activeConnectionId?.(input.candidate.childPeerId) === + input.candidate.connectionId + ) { this.discard(); return false; } - this.startPrepared(input.revision, peerId, input); - return true; - } - - prepareSelectedTurn( - message: SelectedEdgeTurn, - input: Pick, - now = Date.now(), - ): boolean { - if ( - this.preparedRevision !== message.revision || - !this.plannedChildPeerIds.includes(message.viewerPeerId) || - Date.parse(message.expiresAt) <= now - ) { - return false; + if (input.candidate.transport === "direct") { + this.startPrepared(input.revision, input.candidate, input); } - const prepared = this.prepared; - if (prepared && prepared.peerId !== message.viewerPeerId) { - return false; - } - if (prepared?.peer.connectionId === message.newConnectionId) { - return !prepared.failed; - } - const oldConnectionId = - prepared?.peer.connectionId ?? - this.events.activeConnectionId?.(message.viewerPeerId); - if (oldConnectionId !== message.oldConnectionId) { - return false; - } - this.discardPeer(); - this.startPrepared(message.revision, message.viewerPeerId, input, message); return true; } private startPrepared( revision: number, - peerId: string, + candidate: PreparedRouteCandidate, input: Pick, - selectedTurn?: SelectedEdgeTurn, ): void { let peer: HostPeer; peer = new HostPeer( - peerId, - selectedTurn - ? { iceServers: [selectedTurn.iceServer] } - : input.iceConfig, + candidate.childPeerId, + input.iceConfig, input.stream, input.profile, { sendSignal: (targetPeerId, payload) => - this.signalingPeer === peer && targetPeerId === peerId + this.signalingPeer === peer && + targetPeerId === candidate.childPeerId && + payload.connectionId === candidate.connectionId ? this.events.sendSignal(targetPeerId, payload) : false, onUpdate: (snapshot) => { @@ -146,16 +113,17 @@ export class HostProvisionalChild { } }, }, - selectedTurn !== undefined, - selectedTurn?.newConnectionId, + false, + candidate.connectionId, ); this.signalingPeer = peer; this.prepared = { revision, - peerId, + candidate: { ...candidate }, peer, failed: false, - replacesConnectionId: selectedTurn?.oldConnectionId ?? null, + replacesConnectionId: + this.events.activeConnectionId?.(candidate.childPeerId) ?? null, }; void peer .start() @@ -172,7 +140,8 @@ export class HostProvisionalChild { return Boolean( prepared && !prepared.failed && - prepared.peerId === fromPeerId && + prepared.candidate.childPeerId === fromPeerId && + prepared.candidate.connectionId === payload.connectionId && prepared.peer.connectionId === payload.connectionId, ); } @@ -212,22 +181,28 @@ export class HostProvisionalChild { activate(input: HostProvisionalInput): HostPreparedChildActivation { const prepared = this.prepared; const currentConnectionId = prepared - ? this.events.activeConnectionId?.(prepared.peerId) ?? null + ? this.events.activeConnectionId?.(prepared.candidate.childPeerId) ?? null : null; if ( prepared && !prepared.failed && this.preparedRevision === input.revision && prepared.revision === input.revision && - input.assignment.childPeerIds.includes(prepared.peerId) && - (!input.activeChildPeerIds.includes(prepared.peerId) || + this.preparedCandidate?.connectionId === prepared.candidate.connectionId && + input.assignment.childPeerIds.includes(prepared.candidate.childPeerId) && + (!input.activeChildPeerIds.includes(prepared.candidate.childPeerId) || prepared.replacesConnectionId === currentConnectionId) ) { this.prepared = null; this.preparedRevision = null; + this.preparedCandidate = null; this.plannedChildPeerIds = []; this.promotedPeer = prepared.peer; - return { kind: "promote", peerId: prepared.peerId, peer: prepared.peer }; + return { + kind: "promote", + peerId: prepared.candidate.childPeerId, + peer: prepared.peer, + }; } this.discard(); return { kind: "ordinary" }; @@ -235,6 +210,7 @@ export class HostProvisionalChild { discard(): void { this.preparedRevision = null; + this.preparedCandidate = null; this.plannedChildPeerIds = []; this.discardPeer(); } @@ -267,10 +243,12 @@ export class HostProvisionalChild { } function validPlannedHostChildren( - input: HostProvisionalInput, + input: HostProvisionalPrepareInput, ): string[] | null { const childPeerIds = [...new Set(input.assignment.childPeerIds)]; if ( + input.candidate.transport !== "direct" || + !childPeerIds.includes(input.candidate.childPeerId) || childPeerIds.length !== input.assignment.childPeerIds.length || childPeerIds.length < input.activeChildPeerIds.length || childPeerIds.length > input.activeChildPeerIds.length + 1 || @@ -287,6 +265,17 @@ function validPlannedHostChildren( return childPeerIds; } +function sameCandidate( + left: PreparedRouteCandidate | null, + right: PreparedRouteCandidate, +): boolean { + return ( + left?.childPeerId === right.childPeerId && + left.connectionId === right.connectionId && + left.transport === right.transport + ); +} + function samePeerIds(left: readonly string[], right: readonly string[]): boolean { return ( left.length === right.length && diff --git a/src/client/media/host-sfu-route.ts b/src/client/media/host-sfu-route.ts index 776a228..0c29de6 100644 --- a/src/client/media/host-sfu-route.ts +++ b/src/client/media/host-sfu-route.ts @@ -193,8 +193,15 @@ export class HostSfuRoute { const token = this.route.token(); const assignment = this.route.getPlannedAssignment(); const phase = this.route.getPhase(); + const candidate = this.route.getPreparedCandidate(); const publicationGeneration = assignment?.sfuPublicationGeneration; - if (!token || !phase || !assignment || !publicationGeneration) { + if ( + !token || + !phase || + !assignment || + !publicationGeneration || + (phase === "prepare" && candidate?.transport !== "sfu") + ) { return; } this.lastConfig = message; @@ -228,7 +235,7 @@ export class HostSfuRoute { const selectedEdgeTurn = this.selectedEdgeTurn?.revision === message.revision && this.selectedEdgeTurn.publicationGeneration === publicationGeneration && - this.selectedEdgeTurn.oldConnectionId === publicationGeneration + this.selectedEdgeTurn.newConnectionId === candidate?.connectionId ? this.selectedEdgeTurn : null; this.selectedEdgeTurn = null; @@ -299,6 +306,8 @@ export class HostSfuRoute { this.closed || !this.route.acceptsConfig(message.revision) || this.route.getPhase() !== "prepare" || + this.route.getPreparedCandidate()?.transport !== "sfu" || + this.route.getPreparedCandidate()?.connectionId !== message.newConnectionId || Date.parse(message.expiresAt) <= Date.now() ) { return false; @@ -318,12 +327,6 @@ export class HostSfuRoute { if (pending?.connectionId === message.newConnectionId) { return !pending.failed; } - if ( - (pending?.connectionId ?? publicationGeneration) !== - message.oldConnectionId - ) { - return false; - } this.clearPending(); this.selectedEdgeTurn = message; if (this.lastConfig?.revision === message.revision) { diff --git a/src/client/media/route-transition.ts b/src/client/media/route-transition.ts index 17497e1..27e3318 100644 --- a/src/client/media/route-transition.ts +++ b/src/client/media/route-transition.ts @@ -2,13 +2,21 @@ import type { ClientMessage, MediaRoutePhase, ParticipantRouteAssignment, + PreparedRouteCandidate, } from "../../shared/protocol"; -export interface RouteUpdateInput { - revision: number; - phase: MediaRoutePhase; - assignment: ParticipantRouteAssignment; -} +export type RouteUpdateInput = + | { + revision: number; + phase: "prepare"; + assignment: ParticipantRouteAssignment; + candidate: PreparedRouteCandidate; + } + | { + revision: number; + phase: "active"; + assignment: ParticipantRouteAssignment; + }; export interface RouteOperationToken { revision: number; @@ -35,6 +43,7 @@ export class MediaRouteTransition { private plannedAssignment: ParticipantRouteAssignment | null = null; private activeAssignment: ParticipantRouteAssignment | null = null; private mediaAssignment: ParticipantRouteAssignment | null = null; + private preparedCandidate: PreparedRouteCandidate | null = null; private generation = 0; accept(update: RouteUpdateInput): RouteUpdateResult { @@ -45,6 +54,9 @@ export class MediaRouteTransition { if ( !this.plannedAssignment || !sameAssignment(this.plannedAssignment, update.assignment) || + (this.phase === "prepare" && + update.phase === "prepare" && + !sameCandidate(this.preparedCandidate, update.candidate)) || (this.phase === "active" && update.phase === "prepare") ) { return "stale"; @@ -57,6 +69,8 @@ export class MediaRouteTransition { this.revision = update.revision; this.phase = update.phase; this.plannedAssignment = cloneAssignment(update.assignment); + this.preparedCandidate = + update.phase === "prepare" ? { ...update.candidate } : null; if (update.phase === "active") { this.activeAssignment = cloneAssignment(update.assignment); } @@ -113,16 +127,32 @@ export class MediaRouteTransition { return this.mediaAssignment ? cloneAssignment(this.mediaAssignment) : null; } + getPreparedCandidate(): PreparedRouteCandidate | null { + return this.preparedCandidate ? { ...this.preparedCandidate } : null; + } + reset(): void { this.revision = -1; this.phase = null; this.plannedAssignment = null; this.activeAssignment = null; this.mediaAssignment = null; + this.preparedCandidate = null; this.generation += 1; } } +function sameCandidate( + left: PreparedRouteCandidate | null, + right: PreparedRouteCandidate, +): boolean { + return ( + left?.childPeerId === right.childPeerId && + left.connectionId === right.connectionId && + left.transport === right.transport + ); +} + export function reportActivePeerRouteFailure( route: MediaRouteTransition, parentPeerId: string, diff --git a/src/client/media/viewer-sfu-route.ts b/src/client/media/viewer-sfu-route.ts index 515c560..2fda69d 100644 --- a/src/client/media/viewer-sfu-route.ts +++ b/src/client/media/viewer-sfu-route.ts @@ -2,6 +2,7 @@ import type { ClientMessage, MediaRoutePhase, ParticipantRouteAssignment, + PreparedRouteCandidate, ServerMessage, } from "../../shared/protocol"; import { SfuSubscriber } from "../sfu/subscriber"; @@ -40,9 +41,14 @@ interface ViewerSfuRouteEvents { assignment: ParticipantRouteAssignment, revision?: number, ) => boolean | void | Promise; - preparePeer?: (assignment: ParticipantRouteAssignment | null, revision?: number) => void; + preparePeer?: ( + assignment: ParticipantRouteAssignment | null, + revision?: number, + candidate?: PreparedRouteCandidate, + ) => void; prepareChild?: ( - childPeerIds: readonly string[] | null, + candidate: PreparedRouteCandidate | null, + childPeerIds?: readonly string[], revision?: number, ) => void; activateChildren?: ( @@ -83,7 +89,10 @@ export class ViewerSfuRoute { private paused = false; private closed = false; - constructor(private readonly events: ViewerSfuRouteEvents) {} + constructor( + private readonly viewerPeerId: string, + private readonly events: ViewerSfuRouteEvents, + ) {} accept(update: RouteUpdateInput, acknowledge = true): RouteUpdateResult { if (this.closed) { @@ -112,12 +121,21 @@ export class ViewerSfuRoute { } } if (update.phase === "prepare") { - this.events.prepareChild?.(update.assignment.childPeerIds, update.revision); - const mediaUpstream = this.route.getMediaAssignment()?.upstream; + const candidate = update.candidate; + const ownsUpstreamCandidate = candidate.childPeerId === this.viewerPeerId; + const ownsChildCandidate = + candidate.transport !== "sfu" && + update.assignment.childPeerIds.includes(candidate.childPeerId); + this.events.prepareChild?.( + ownsChildCandidate ? candidate : null, + ownsChildCandidate ? update.assignment.childPeerIds : undefined, + ownsChildCandidate ? update.revision : undefined, + ); const needsPeerCandidate = + ownsUpstreamCandidate && + candidate.transport !== "sfu" && update.assignment.upstream.kind === "peer" && - (mediaUpstream?.kind !== "peer" || - mediaUpstream.peerId !== update.assignment.upstream.peerId); + candidate.connectionId.length > 0; const replacedPeerCandidate = this.pendingPeerRevision !== null; if (replacedPeerCandidate) { this.pendingPeerRevision = null; @@ -125,7 +143,7 @@ export class ViewerSfuRoute { } if (needsPeerCandidate && update.assignment.upstream.kind === "peer") { this.pendingPeerRevision = update.revision; - this.events.preparePeer?.(update.assignment, update.revision); + this.events.preparePeer?.(update.assignment, update.revision, candidate); } else if ( update.assignment.upstream.kind !== "peer" && !replacedPeerCandidate @@ -135,7 +153,11 @@ export class ViewerSfuRoute { if (this.pending?.revision !== update.revision) { this.clearPending(); } - if (update.assignment.upstream.kind !== "sfu") { + if ( + !ownsUpstreamCandidate || + candidate.transport !== "sfu" || + update.assignment.upstream.kind !== "sfu" + ) { this.clearPending(); } return result; @@ -155,21 +177,6 @@ export class ViewerSfuRoute { return result; } - beginSelectedPeerCandidate(revision: number, parentPeerId: string): boolean { - const assignment = this.route.getPlannedAssignment(); - if ( - this.paused || - this.route.getRevision() !== revision || - this.route.getPhase() !== "prepare" || - assignment?.upstream.kind !== "peer" || - assignment.upstream.peerId !== parentPeerId - ) { - return false; - } - this.pendingPeerRevision = revision; - return true; - } - setPaused(paused: boolean): void { this.paused = paused; if (paused) { @@ -253,7 +260,15 @@ export class ViewerSfuRoute { const token = this.route.token(); const assignment = this.route.getPlannedAssignment(); const phase = this.route.getPhase(); - if (!token || !phase || assignment?.upstream.kind !== "sfu") { + const candidate = this.route.getPreparedCandidate(); + if ( + !token || + !phase || + assignment?.upstream.kind !== "sfu" || + (phase === "prepare" && + (candidate?.childPeerId !== this.viewerPeerId || + candidate.transport !== "sfu")) + ) { return; } @@ -555,15 +570,7 @@ export class ViewerSfuRoute { const changed = slot.mediaAvailable !== available; slot.mediaAvailable = available; this.events.onSfuVideoAvailability?.(available); - if (!changed) { - return; - } - if (!available) { - this.events.send({ - type: "route-media-unavailable", - revision: slot.revision, - }); - } + if (!changed || this.paused) return; } private commitMedia(token: RouteOperationToken): void { diff --git a/src/client/pages/HostPage.tsx b/src/client/pages/HostPage.tsx index 9bc222b..e784b4f 100644 --- a/src/client/pages/HostPage.tsx +++ b/src/client/pages/HostPage.tsx @@ -27,6 +27,7 @@ import { type CreateRoomResponse, type IceConfig, type ParticipantPresenceEntry, + type PreparedRouteCandidate, type ServerMessage, type ViewerAccessPolicy, } from "../../shared/protocol"; @@ -272,6 +273,7 @@ export function HostPage({ onAuthorizationRequired }: HostPageProps = {}) { const retiredConnectionsRef = useRef(new Map()); const hostProvisionalChildRef = useRef(null); const activeHostChildPeerIdsRef = useRef([]); + const endpointMediaCopyCapacityRef = useRef(MAX_ENDPOINT_MEDIA_CHILDREN); const hostPeerIdRef = useRef(null); const viewerQualityEvidenceRef = useRef( new Map(), @@ -411,6 +413,7 @@ export function HostPage({ onAuthorizationRequired }: HostPageProps = {}) { hostProvisionalChildRef.current?.discard(); hostProvisionalChildRef.current = null; activeHostChildPeerIdsRef.current = []; + endpointMediaCopyCapacityRef.current = MAX_ENDPOINT_MEDIA_CHILDREN; hostPeerIdRef.current = null; viewerQualityEvidenceTimersRef.current.forEach((timer) => window.clearTimeout(timer), @@ -526,6 +529,7 @@ export function HostPage({ onAuthorizationRequired }: HostPageProps = {}) { hostProvisionalChildRef.current?.discard(); hostProvisionalChildRef.current = null; activeHostChildPeerIdsRef.current = []; + endpointMediaCopyCapacityRef.current = MAX_ENDPOINT_MEDIA_CHILDREN; hostPeerIdRef.current = null; void hostSfuRouteRef.current?.disconnect(); hostSfuRouteRef.current = null; @@ -909,6 +913,7 @@ export function HostPage({ onAuthorizationRequired }: HostPageProps = {}) { function prepareHostChild( revision: number, assignment: HostRouteAssignment, + candidate: PreparedRouteCandidate, generation: number, ): boolean { const stream = streamRef.current; @@ -950,8 +955,9 @@ export function HostPage({ onAuthorizationRequired }: HostPageProps = {}) { return hostProvisionalChildRef.current.prepare({ revision, assignment, + candidate, activeChildPeerIds: activeHostChildPeerIdsRef.current, - maxMediaEdges: MAX_ENDPOINT_MEDIA_CHILDREN, + maxMediaEdges: endpointMediaCopyCapacityRef.current, iceConfig, stream, profile: qualitySettingsRef.current, @@ -969,7 +975,7 @@ export function HostPage({ onAuthorizationRequired }: HostPageProps = {}) { revision, assignment, activeChildPeerIds, - maxMediaEdges: MAX_ENDPOINT_MEDIA_CHILDREN, + maxMediaEdges: endpointMediaCopyCapacityRef.current, }) ?? { kind: "ordinary" as const }; if (activation.kind === "promote") { hostProvisionalChildRef.current = null; @@ -1106,7 +1112,7 @@ export function HostPage({ onAuthorizationRequired }: HostPageProps = {}) { reconcileBoundedMediaChildren( peersRef.current.keys(), childPeerIds, - MAX_ENDPOINT_MEDIA_CHILDREN, + endpointMediaCopyCapacityRef.current, removePeer, (peerId) => { void startPeer(peerId, generation).catch((error: unknown) => { @@ -1129,6 +1135,7 @@ export function HostPage({ onAuthorizationRequired }: HostPageProps = {}) { if (message.type === "authenticated" && message.role === "host") { discardPreparedHostChild(); hostPeerIdRef.current = message.peerId; + endpointMediaCopyCapacityRef.current = message.endpointMediaCopyCapacity; setViewerAccessUpdating(false); clearAllViewerQualityEvidence(); setSfuStandbyUrl( @@ -1193,20 +1200,6 @@ export function HostPage({ onAuthorizationRequired }: HostPageProps = {}) { } return; } - const iceConfig = iceConfigRef.current; - const activeStream = streamRef.current; - if ( - peerAssistedRef.current && - message.parentPeerId === hostPeerIdRef.current && - iceConfig && - activeStream - ) { - hostProvisionalChildRef.current?.prepareSelectedTurn(message, { - iceConfig, - stream: activeStream, - profile: qualitySettingsRef.current, - }); - } return; } if (message.type === "viewer-access-updated") { @@ -1249,8 +1242,20 @@ export function HostPage({ onAuthorizationRequired }: HostPageProps = {}) { } if (message.type === "route-update") { if (peerAssistedRef.current) { + const route = ensureHostSfuRoute(generation); + const accepted = route.accept(message); + if (accepted === "stale") return; if (message.phase === "prepare") { - prepareHostChild(message.revision, message.assignment, generation); + if (message.candidate.transport === "direct") { + prepareHostChild( + message.revision, + message.assignment, + message.candidate, + generation, + ); + } else { + discardPreparedHostChild(); + } } else { activatePreparedHostChild(message.revision, message.assignment); } @@ -1261,10 +1266,7 @@ export function HostPage({ onAuthorizationRequired }: HostPageProps = {}) { activeRouteRevisionRef.current = message.revision; clearAllViewerQualityEvidence(); } - const route = ensureHostSfuRoute(generation); - void route - .acceptAndWait(message) - .then(() => showHostSfuQualityWarning(route, generation)); + showHostSfuQualityWarning(route, generation); } return; } diff --git a/src/client/pages/ViewerPage.tsx b/src/client/pages/ViewerPage.tsx index 32a859e..e924f5b 100644 --- a/src/client/pages/ViewerPage.tsx +++ b/src/client/pages/ViewerPage.tsx @@ -42,6 +42,7 @@ import { clearViewerGrant, getStableClientId } from "../lib/session"; import { SignalingClient } from "../lib/signaling"; import { labelParticipantSnapshot } from "../lib/viewer-presence"; import { downloadDiagnosticReport, type DiagnosticConnectionInput } from "../lib/diagnostic-export"; +import { DecodedFrameStallDetector } from "../media/decoded-frame-stall"; import type { QualitySettings } from "../media/quality"; import { ParentEdgeQualityEvidenceReporter, @@ -105,7 +106,8 @@ interface PendingPeerRoute { peer: ViewerPeer | null; readySent: boolean; selectedTurn: SelectedEdgeTurn | null; - candidateConnectionId: string | null; + candidateConnectionId: string; + transport: "direct" | "selected-turn"; stream: MediaStream | null; snapshot: PeerSnapshot | null; } @@ -254,9 +256,11 @@ export function ViewerPage({ roomId, viewerGrant }: ViewerPageProps) { let viewerAuthenticated = false; let currentIceConfig: IceConfig | null = null; let currentHostOnline = false; + let currentHostPaused = false; let peerAssisted = false; let currentPeerId: string | null = null; let currentRouteRevision = 0; + let endpointMediaCopyCapacity = MAX_ENDPOINT_MEDIA_CHILDREN; let viewerAuthorizationGeneration: string | null = null; let currentQualitySettings: QualitySettings = DEFAULT_QUALITY_SETTINGS; let currentAssignment: MediaAssignment = { @@ -270,6 +274,7 @@ export function ViewerPage({ roomId, viewerGrant }: ViewerPageProps) { Extract > = []; let pendingPeer: PendingPeerRoute | null = null; + const decodedFrameStall = new DecodedFrameStallDetector(); const messageAuthority = new ViewerMessageAuthority(); let sfuStandbyPrewarmer: SfuStandbyPrewarmer | null = null; let relayChildEvidenceCurrent: ViewerQualityEvidencePresentation | null = @@ -448,6 +453,7 @@ export function ViewerPage({ roomId, viewerGrant }: ViewerPageProps) { } }, }, + endpointMediaCopyCapacity, ); viewerRelay.setChildren(currentAssignment.childPeerIds); return viewerRelay; @@ -486,6 +492,37 @@ export function ViewerPage({ roomId, viewerGrant }: ViewerPageProps) { probe?.peer?.dispose(); } + function observeActiveDecodedFrames( + route: "peer" | "sfu", + identity: string, + framesDecodedDelta: number | null, + connectionId?: string, + ): void { + if ( + !peerAssisted || + currentHostPaused || + !decodedFrameStall.observe( + `${route}:${currentRouteRevision}:${identity}`, + framesDecodedDelta, + ) + ) { + return; + } + if (route === "peer" && connectionId) { + signal.send({ + type: "route-failed", + revision: currentRouteRevision, + phase: "active", + connectionId, + }); + } else if (route === "sfu") { + signal.send({ + type: "route-media-unavailable", + revision: currentRouteRevision, + }); + } + } + function pendingPeerHasDecodedFrame( probe: PendingPeerRoute, ): probe is PendingPeerRoute & { peer: ViewerPeer; snapshot: PeerSnapshot } { @@ -515,7 +552,7 @@ export function ViewerPage({ roomId, viewerGrant }: ViewerPageProps) { return viewerSfuRoute; } let route: ViewerSfuRoute; - route = new ViewerSfuRoute({ + route = new ViewerSfuRoute(currentPeerId!, { activatePeer: async (assignment, revision) => { if (!active || viewerSfuRoute !== route) { return; @@ -565,19 +602,26 @@ export function ViewerPage({ roomId, viewerGrant }: ViewerPageProps) { await drainPreparedParentSignals(assignment.upstream.peerId); } }, - preparePeer: (assignment, revision) => { + preparePeer: (assignment, revision, candidate) => { if (!active || viewerSfuRoute !== route) { return; } discardPendingPeer(); - if (revision !== undefined && assignment?.upstream.kind === "peer") { + if ( + revision !== undefined && + candidate && + candidate.childPeerId === currentPeerId && + candidate.transport !== "sfu" && + assignment?.upstream.kind === "peer" + ) { pendingPeer = { revision, parentPeerId: assignment.upstream.peerId, peer: null, readySent: false, selectedTurn: null, - candidateConnectionId: null, + candidateConnectionId: candidate.connectionId, + transport: candidate.transport, stream: null, snapshot: null, }; @@ -588,12 +632,12 @@ export function ViewerPage({ roomId, viewerGrant }: ViewerPageProps) { : null, ); }, - prepareChild: (childPeerIds, revision) => { + prepareChild: (candidate, childPeerIds, revision) => { if (!active || viewerSfuRoute !== route) { return; } - if (childPeerIds && revision !== undefined) { - ensureViewerRelay()?.prepareChild(revision, childPeerIds); + if (candidate && childPeerIds && revision !== undefined) { + ensureViewerRelay()?.prepareChild(revision, candidate, childPeerIds); } else { viewerRelay?.discardPreparedChild(); } @@ -618,12 +662,19 @@ export function ViewerPage({ roomId, viewerGrant }: ViewerPageProps) { parentPeerId: currentAssignment.parentPeerId, childPeerIds: [...childPeerIds], }, - MAX_ENDPOINT_MEDIA_CHILDREN, + endpointMediaCopyCapacity, ); reconcileRelayChildren(previousChildPeerIds, revision); }, onSfuUpdate: (metrics) => { if (active && viewerSfuRoute === route) { + if (metrics) { + observeActiveDecodedFrames( + "sfu", + String(currentRouteRevision), + metrics.intervalFramesDecoded, + ); + } setSfuUpstream((current) => metrics && current ? { ...current, metrics } : null, ); @@ -640,8 +691,9 @@ export function ViewerPage({ roomId, viewerGrant }: ViewerPageProps) { if (!active || viewerSfuRoute !== route || available) { return; } - setSfuUpstream(null); - viewerRelay?.stop(); + setSfuUpstream((current) => + current ? { ...current, connectionState: "reconnecting" } : current, + ); setStatusText( currentHostOnline ? "正在恢复连接" : "等待开始分享", ); @@ -653,7 +705,7 @@ export function ViewerPage({ roomId, viewerGrant }: ViewerPageProps) { const previousChildPeerIds = currentAssignment.childPeerIds; currentAssignment = limitMediaAssignment( { parentPeerId: null, childPeerIds: assignment.childPeerIds }, - MAX_ENDPOINT_MEDIA_CHILDREN, + endpointMediaCopyCapacity, ); reconcileRelayChildren(previousChildPeerIds); const relay = ensureViewerRelay(); @@ -735,7 +787,7 @@ export function ViewerPage({ roomId, viewerGrant }: ViewerPageProps) { function applyMediaAssignment(assignment: MediaAssignment, preserveUpstream = false): void { const nextAssignment = limitMediaAssignment( assignment, - MAX_ENDPOINT_MEDIA_CHILDREN, + endpointMediaCopyCapacity, ); const previousParentId = currentAssignment.parentPeerId; const previousChildPeerIds = currentAssignment.childPeerIds; @@ -772,10 +824,16 @@ export function ViewerPage({ roomId, viewerGrant }: ViewerPageProps) { }, onUpdate: (snapshot) => { if (pendingPeer === probe) { + if (snapshot.connectionId !== probe.candidateConnectionId) return; probe.snapshot = snapshot; - probe.candidateConnectionId = snapshot.connectionId; provePendingPeer(); } else if (active && peerRef.current === peer) { + observeActiveDecodedFrames( + "peer", + `${probe.parentPeerId}:${snapshot.connectionId}`, + snapshot.metrics.intervalFramesDecoded, + snapshot.connectionId, + ); qualityEvidenceReporter.offer(snapshot, currentRouteRevision); setPeerSnapshot(snapshot); setStatusText( @@ -793,7 +851,6 @@ export function ViewerPage({ roomId, viewerGrant }: ViewerPageProps) { phase: "prepare", connectionId, }); - probe.candidateConnectionId = connectionId; probe.peer = null; probe.stream = null; probe.snapshot = null; @@ -806,7 +863,7 @@ export function ViewerPage({ roomId, viewerGrant }: ViewerPageProps) { : true; }, }, - probe.selectedTurn !== null, + probe.transport === "selected-turn", ); probe.peer = peer; return peer; @@ -846,6 +903,12 @@ export function ViewerPage({ roomId, viewerGrant }: ViewerPageProps) { }, onUpdate: (snapshot) => { if (active) { + observeActiveDecodedFrames( + "peer", + `${snapshot.peerId}:${snapshot.connectionId}`, + snapshot.metrics.intervalFramesDecoded, + snapshot.connectionId, + ); qualityEvidenceReporter.offer(snapshot, currentRouteRevision); if ( !currentHostOnline && @@ -890,6 +953,7 @@ export function ViewerPage({ roomId, viewerGrant }: ViewerPageProps) { setViewerPasswordError(null); clearRelayChildEvidence(); currentPeerId = message.peerId; + endpointMediaCopyCapacity = message.endpointMediaCopyCapacity; viewerAuthorizationGeneration = message.viewerAuthorizationGeneration; setSfuStandbyUrl( @@ -930,6 +994,8 @@ export function ViewerPage({ roomId, viewerGrant }: ViewerPageProps) { currentHostOnline = message.hostOnline; setHostOnline(message.hostOnline); const sharingPaused = message.hostPaused ?? false; + currentHostPaused = sharingPaused; + decodedFrameStall.setPaused(sharingPaused); setHostPaused(sharingPaused); if (nextPeerAssisted && "qualitySettings" in message) { currentQualitySettings = message.qualitySettings; @@ -997,54 +1063,22 @@ export function ViewerPage({ roomId, viewerGrant }: ViewerPageProps) { ); return; } - let probe = pendingPeer; - if ( - !viewerSfuRoute?.beginSelectedPeerCandidate( - message.revision, - message.parentPeerId, - ) - ) { + const probe = pendingPeer; + if (probe?.selectedTurn?.newConnectionId === message.newConnectionId) { return; } - if (!probe) { - probe = pendingPeer = { - revision: message.revision, - parentPeerId: message.parentPeerId, - peer: null, - readySent: false, - selectedTurn: null, - candidateConnectionId: null, - stream: null, - snapshot: null, - }; - prepareParent(message.parentPeerId); - } - if (probe.selectedTurn?.newConnectionId === message.newConnectionId) { - return; - } - const activeIdentity = peerRef.current?.getConnectionIdentity(); - const oldConnectionId = - probe?.candidateConnectionId ?? - probe?.peer?.getConnectionIdentity()?.connectionId ?? - (activeIdentity?.parentPeerId === message.parentPeerId - ? activeIdentity.connectionId - : null); if ( message.viewerPeerId !== currentPeerId || probe?.revision !== message.revision || probe.parentPeerId !== message.parentPeerId || + probe.transport !== "selected-turn" || + probe.candidateConnectionId !== message.newConnectionId || Date.parse(message.expiresAt) <= Date.now() || - oldConnectionId !== message.oldConnectionId + probe.peer !== null ) { return; } - probe.peer?.dispose(); - probe.peer = null; - probe.readySent = false; probe.selectedTurn = message; - probe.candidateConnectionId = null; - probe.stream = null; - probe.snapshot = null; return; } if (message.type === "route-update") { @@ -1094,6 +1128,13 @@ export function ViewerPage({ roomId, viewerGrant }: ViewerPageProps) { } if (message.type === "signal") { if (peerAssisted && pendingPeer?.parentPeerId === message.fromPeerId) { + if ( + message.payload.connectionId === pendingPeer.candidateConnectionId && + pendingPeer.transport === "selected-turn" && + !pendingPeer.selectedTurn + ) { + return; + } if (currentAssignment.parentPeerId !== message.fromPeerId) { await ensurePendingPeerRoute()?.acceptSignal( message.fromPeerId, @@ -1104,8 +1145,7 @@ export function ViewerPage({ roomId, viewerGrant }: ViewerPageProps) { const activeIdentity = peerRef.current?.getConnectionIdentity(); const owner = exactPeerSignalOwner( message.payload.connectionId, - pendingPeer.selectedTurn?.newConnectionId ?? - pendingPeer.candidateConnectionId, + pendingPeer.candidateConnectionId, activeIdentity?.parentPeerId === message.fromPeerId ? activeIdentity.connectionId : null, @@ -1171,6 +1211,8 @@ export function ViewerPage({ roomId, viewerGrant }: ViewerPageProps) { } if (message.type === "host-status") { currentHostOnline = message.online; + currentHostPaused = message.paused; + decodedFrameStall.setPaused(message.paused); setHostOnline(message.online); setHostPaused(message.paused); viewerSfuRoute?.setPaused(message.paused); @@ -1189,6 +1231,8 @@ export function ViewerPage({ roomId, viewerGrant }: ViewerPageProps) { setSfuStandbyUrl(null); setAssignedRoute(null); currentHostOnline = false; + currentHostPaused = false; + decodedFrameStall.reset(); clearViewerSfuRoute(); clearPeerState(); clearHostPresence(); diff --git a/src/client/webrtc/viewer-relay.ts b/src/client/webrtc/viewer-relay.ts index 00d69c1..b9fd1d7 100644 --- a/src/client/webrtc/viewer-relay.ts +++ b/src/client/webrtc/viewer-relay.ts @@ -1,5 +1,6 @@ import type { IceConfig, + PreparedRouteCandidate, ServerMessage, SignalPayload, } from "../../shared/protocol"; @@ -22,6 +23,7 @@ type SelectedEdgeTurn = Extract< interface PreparedChild { revision: number; childPeerId: string; + candidate: PreparedRouteCandidate; peer: HostPeer; failed: boolean; replacesConnectionId: string | null; @@ -34,6 +36,7 @@ export class ViewerRelay { private readonly snapshots = new Map(); private preparedChild: PreparedChild | null = null; private preparedRevision: number | null = null; + private preparedCandidate: PreparedRouteCandidate | null = null; private plannedChildPeerIds: string[] = []; private syncQueue = Promise.resolve(); private disposed = false; @@ -42,6 +45,7 @@ export class ViewerRelay { private iceConfig: IceConfig, private desiredProfile: QualityProfile, private readonly events: ViewerRelayEvents, + private readonly maxMediaEdges = MAX_ENDPOINT_MEDIA_CHILDREN, ) {} getSnapshot(childPeerId?: string): PeerSnapshot | null { @@ -60,38 +64,37 @@ export class ViewerRelay { routeRevision: number, now = Date.now(), ): boolean { - const stream = this.stream; if ( this.disposed || - !stream || message.parentPeerId !== parentPeerId || message.revision !== routeRevision || Date.parse(message.expiresAt) <= now || this.preparedRevision !== routeRevision || - !this.plannedChildPeerIds.includes(message.viewerPeerId) + this.preparedCandidate?.transport !== "selected-turn" || + this.preparedCandidate.childPeerId !== message.viewerPeerId || + this.preparedCandidate.connectionId !== message.newConnectionId ) { return false; } const prepared = this.preparedChild; - if (prepared && prepared.childPeerId !== message.viewerPeerId) { - return false; - } if (prepared?.peer.connectionId === message.newConnectionId) { return !prepared.failed; } - const oldConnectionId = - prepared?.peer.connectionId ?? - this.peers.get(message.viewerPeerId)?.connectionId; - if (oldConnectionId !== message.oldConnectionId) { - return false; - } + const stream = this.stream; + if (!stream) return false; this.discardPreparedPeer(); - this.startPreparedChild(message.revision, message.viewerPeerId, stream, message); + this.startPreparedChild( + message.revision, + this.preparedCandidate, + stream, + message, + ); return true; } prepareChild( revision: number, + candidate: PreparedRouteCandidate, plannedChildPeerIds: readonly string[], ): boolean { const stream = this.stream; @@ -99,10 +102,12 @@ export class ViewerRelay { if ( this.disposed || !stream || + candidate.transport === "sfu" || + !planned.includes(candidate.childPeerId) || planned.length !== plannedChildPeerIds.length || planned.length < this.childPeerIds.length || planned.length > this.childPeerIds.length + 1 || - planned.length > MAX_ENDPOINT_MEDIA_CHILDREN || + planned.length > this.maxMediaEdges || this.childPeerIds.some((peerId) => !planned.includes(peerId)) ) { this.discardPreparedChild(); @@ -110,6 +115,7 @@ export class ViewerRelay { } if ( this.preparedRevision === revision && + sameCandidate(this.preparedCandidate, candidate) && samePeerIds(this.plannedChildPeerIds, planned) ) { return this.preparedChild ? !this.preparedChild.failed : true; @@ -117,14 +123,11 @@ export class ViewerRelay { this.discardPreparedChild(); this.preparedRevision = revision; + this.preparedCandidate = { ...candidate }; this.plannedChildPeerIds = planned; - const childPeerId = planned.find( - (peerId) => !this.childPeerIds.includes(peerId), - ); - if (!childPeerId) { - return true; + if (candidate.transport === "direct") { + this.startPreparedChild(revision, candidate, stream); } - this.startPreparedChild(revision, childPeerId, stream); return true; } @@ -142,11 +145,13 @@ export class ViewerRelay { if ( prepared?.revision === revision && !prepared.failed && + prepared.candidate.connectionId === prepared.peer.connectionId && childPeerIds.includes(prepared.childPeerId) && (extendsActive || replacesActive) ) { this.preparedChild = null; this.preparedRevision = null; + this.preparedCandidate = null; this.plannedChildPeerIds = []; this.peers.set(prepared.childPeerId, prepared.peer); replacedPeer?.dispose(); @@ -160,6 +165,7 @@ export class ViewerRelay { discardPreparedChild(): void { this.preparedRevision = null; + this.preparedCandidate = null; this.plannedChildPeerIds = []; this.discardPreparedPeer(); } @@ -183,7 +189,7 @@ export class ViewerRelay { } const nextChildPeerIds = [...new Set(childPeerIds)].slice( 0, - MAX_ENDPOINT_MEDIA_CHILDREN, + this.maxMediaEdges, ); for (const childPeerId of this.childPeerIds) { if (!nextChildPeerIds.includes(childPeerId)) { @@ -330,17 +336,24 @@ export class ViewerRelay { private startPreparedChild( revision: number, - childPeerId: string, + candidate: PreparedRouteCandidate, stream: MediaStream, selectedTurn?: SelectedEdgeTurn, ): void { - const peer = this.createPeer(childPeerId, stream, selectedTurn); + const peer = this.createPeer( + candidate.childPeerId, + stream, + candidate.connectionId, + selectedTurn, + ); this.preparedChild = { revision, - childPeerId, + childPeerId: candidate.childPeerId, + candidate: { ...candidate }, peer, failed: false, - replacesConnectionId: selectedTurn?.oldConnectionId ?? null, + replacesConnectionId: + this.peers.get(candidate.childPeerId)?.connectionId ?? null, }; void peer .start() @@ -470,6 +483,7 @@ export class ViewerRelay { private createPeer( childPeerId: string, stream: MediaStream, + connectionId?: string, selectedTurn?: SelectedEdgeTurn, ): HostPeer { let peer: HostPeer; @@ -511,7 +525,7 @@ export class ViewerRelay { }, }, selectedTurn !== undefined, - selectedTurn?.newConnectionId, + connectionId, ); return peer; } @@ -541,6 +555,17 @@ export class ViewerRelay { } +function sameCandidate( + left: PreparedRouteCandidate | null, + right: PreparedRouteCandidate, +): boolean { + return ( + left?.childPeerId === right.childPeerId && + left.connectionId === right.connectionId && + left.transport === right.transport + ); +} + function samePeerIds(left: readonly string[], right: readonly string[]): boolean { return ( left.length === right.length && diff --git a/src/server/hybrid-media-router.ts b/src/server/hybrid-media-router.ts index ed02ada..683f208 100644 --- a/src/server/hybrid-media-router.ts +++ b/src/server/hybrid-media-router.ts @@ -1,41 +1,34 @@ import { randomBytes } from "node:crypto"; -import { - MAX_MEDIA_ROUTE_REVISION, - type ClientMessage, - type MediaAssignment, - type MediaRouteUpstream, - type ParentEdgeQualityProof, - type ParticipantRouteAssignment, - type RelayDownstreamEdges, - type Role, - type ServerMessage, +import type { + ClientMessage, + MediaAssignment, + MediaRouteUpstream, + ParticipantRouteAssignment, + PreparedRouteCandidate, + Role, + ServerMessage, } from "../shared/protocol.js"; -import { - assertEndpointMediaCopyCapacity, - countEndpointMediaCopies, - endpointMediaCopyCountFits, -} from "../shared/media-copy-accounting.js"; +import { assertEndpointMediaCopyCapacity } from "../shared/media-copy-accounting.js"; import type { SfuTokenIssuer } from "./livekit-token.js"; +import type { SelectedEdgeTurnConfig } from "./config.js"; +import { + RoomRouteController, + type CandidateCursorGuard, + type CandidatePlan, + type CandidateReservation, + type CandidateTuple, + type OperationSnapshot, + type RouteSnapshot, +} from "./room-route-controller.js"; +import type { RoomStore } from "./room-store.js"; +import type { SfuRoomControl } from "./sfu-room-control.js"; +import { managedSfuRoomName } from "./sfu-room-control.js"; import type { SfuResourceAdmission, SfuResourceFence, + SfuSubscriptionFence, } from "./sfu-resource-admission.js"; -import { - managedSfuRoomName, - type SfuRoomControl, -} from "./sfu-room-control.js"; -import type { SelectedEdgeTurnConfig } from "./config.js"; -import { - MediaRouteController, - type RoomMediaRoute, -} from "./media-route-controller.js"; -import { - MAX_PEER_RELAY_DEPTH, - PeerRelayTopology, - type MediaAssignmentChange, -} from "./peer-relay-topology.js"; -import type { RoomStore } from "./room-store.js"; import { issueSelectedEdgeTurnCredential, type HostSfuIngressTurnIdentity, @@ -46,34 +39,41 @@ import type { TurnAllocationAdmission } from "./turn-allocation-admission.js"; type ErrorCode = Extract["code"]; -const DEFAULT_SFU_PREPARE_TIMEOUT_MS = 20_000; +const DEFAULT_ROUTE_OPERATION_TIMEOUT_MS = 20_000; const DEFAULT_SFU_DRAIN_RETRY_MS = 1_000; const DEFAULT_HOST_OFFLINE_CHECK_MS = 5_000; -const SELECTED_EDGE_TURN_ATTEMPT_TIMEOUT_MS = 30_000; -const VIEWER_QUALITY_BAD_WINDOWS_TO_REASSIGN = 3; -const VIEWER_QUALITY_EVIDENCE_GAP_MS = 5_000; -const VIEWER_QUALITY_REASSIGN_COOLDOWN_MS = 30_000; -const VIEWER_QUALITY_FREEZE_RATIO = 0.5; -const VIEWER_QUALITY_RELAY_FPS_RATIO = 2 / 3; -const VIEWER_QUALITY_MIN_LOSS_PACKETS = 100; -const VIEWER_QUALITY_HIGH_LOSS_RATIO = 0.3; -interface PendingRoutePreparation { - revision: number; - intentPeerId: string; - routeIntent: ViewerRouteIntent; - qualityIntent: { - guard: QualityRouteIntentGuard; - takenOverByRouteFailure: boolean; - } | null; - expectedSessionIds: ReadonlyMap; - grantsIssued: boolean; - hostSfuIngressTurnAttempted: boolean; - healthyReselection: HealthySfuReselectionProbe | null; - peerQualityReselection: PeerQualityReselectionProbe | null; - sfuResourceFence: SfuResourceFence | null; - timer: NodeJS.Timeout; + +interface RouteResourceBase { + released: boolean; } +interface TurnRouteResource extends RouteResourceBase { + kind: "turn"; + fence: SelectedEdgeTurnIdentity; +} + +interface SfuSubscriptionRouteResource extends RouteResourceBase { + kind: "sfu-subscription"; + fence: SfuSubscriptionFence; +} + +interface SfuPublicationRouteResource extends RouteResourceBase { + kind: "sfu-publication"; + fence: SfuResourceFence; + ingressTurn?: TurnRouteResource; +} + +interface OverlapRouteResource extends RouteResourceBase { + kind: "overlap"; + endpointPeerId: string; +} + +type RouteResource = + | TurnRouteResource + | SfuSubscriptionRouteResource + | SfuPublicationRouteResource + | OverlapRouteResource; + interface SfuDrainTask { fence: SfuResourceFence; operation?: Promise; @@ -86,132 +86,36 @@ interface HostOfflineCheck { timer: NodeJS.Timeout; } -interface ViewerRouteIntent { - failedParentPeerId: string | null; - failedParentSessionId?: string; - failedConnectionId?: string; - sessionId: string; - sfuAttempts: number; - selectedTurnAttempted?: boolean; - unavailableReported: boolean; - qualityGuard?: QualityRouteIntentGuard; +interface RoomRuntime { + hostPeerId?: string; + controller?: RoomRouteController; + advertisedCapacityByViewer: Map; + requested: boolean; + pump?: Promise; + deadlineTimer?: NodeJS.Timeout; + refreshKeys: Set; } -interface SelectedPeerEdgeLease { - roomId: string; - currentRouteRevision: number; - shareGeneration: string; - parentPeerId: string; - parentSessionId: string; - viewerPeerId: string; - viewerSessionId: string; - newConnectionId: string; - allocationFence: PeerSelectedEdgeTurnIdentity; - grant: Extract< +interface PreparedCandidate { + reservation: CandidateReservation; + connectionId: string; + publicationGeneration?: string; + publicationConnectionId?: string; + peerGrant?: Extract< ServerMessage, { type: "selected-edge-turn"; edgeKind: "peer-selected" } >; - answered: boolean; - timer: NodeJS.Timeout; -} - -interface SelectedSfuIngressAttempt { - roomId: string; - revision: number; - shareGeneration: string; - hostPeerId: string; - hostSessionId: string; - publicationGeneration: string; - oldConnectionId: string; - newConnectionId: string; - allocationFence: HostSfuIngressTurnIdentity; - answered: boolean; - timer: NodeJS.Timeout; -} - -interface ActiveSelectedSfuIngress { - shareGeneration: string; - hostPeerId: string; - hostSessionId: string; - publicationGeneration: string; - connectionId: string; - allocationFence: HostSfuIngressTurnIdentity; -} - -interface HealthySfuReselectionProbe { - activeRevision: number; - shareGeneration: string; - publicationGeneration: string; - rootPeerId: string; - parentPeerId: string; -} - -interface PeerQualityReselectionProbe { - activeRevision: number; - shareGeneration: string; - excludedParentPeerIds: ReadonlySet; - newParentPeerId: string; - newConnectionId: string | null; -} - -interface DeferredHealthySfuReselection { - activeRevision: number; - shareGeneration: string; - publicationGeneration: string; - rootPeerId: string; - rootSessionId: string; - timer: NodeJS.Timeout | null; -} - -interface QualityRouteIntentGuard { - kind: ViewerQualityBadWindowKind; - viewerSessionId: string; - connectionId: string; - routeRevision: number; - parentPeerId: string; - parentSessionId: string; - exclusionOwnedByQuality: boolean; -} - -type ViewerQualityBadWindowKind = "severe" | "relative-fps"; - -interface ViewerQualityEvidenceState { - viewerSessionId: string; - parentPeerId: string; - parentSessionId: string; - connectionId: string; - routeRevision: number; - lastCorrelatedAtMs: number | null; - badWindowCount: number; - badWindowKind: ViewerQualityBadWindowKind | null; - lastCorrelatedFps: { - correlatedAtMs: number; - framesPerSecond: number | null; - } | null; - pendingViewerEvidence: { - acceptedAtMs: number; - evidence: Extract; - } | null; -} - -interface ForwardedViewerQualityEvidence { - roomId: string; - viewerSessionId: string; - parentSessionId: string; - evidence: Extract; -} - -interface ForwardedParentEdgeQualityEvidence { - roomId: string; - viewerSessionId: string; - parentSessionId: string; - evidence: Extract< - ClientMessage, - { type: "parent-edge-quality-evidence" } + hostIngressGrant?: Extract< + ServerMessage, + { type: "selected-edge-turn"; edgeKind: "host-sfu-ingress" } >; + hostSfuConfig?: Extract; + viewerSfuConfig?: Extract; } -type SfuPrepareResult = "started" | "busy" | "unavailable"; +type PrepareResult = + | { kind: "denied"; serverAdmission: "turn" | "sfu" | false } + | { kind: "ready"; prepared: PreparedCandidate }; export interface SfuFallbackOptions { url: string; @@ -254,8 +158,6 @@ export interface HybridAuthenticationState { routeRevision: number; routeAssignment: ParticipantRouteAssignment; mediaAssignment: MediaAssignment; - assignmentChanges: readonly MediaAssignmentChange[]; - activeRoute?: RoomMediaRoute; } export interface AuthenticatedRouteParticipant { @@ -266,267 +168,106 @@ export interface AuthenticatedRouteParticipant { } export class HybridMediaRouter { - private readonly peerRelayTopology: PeerRelayTopology; - private readonly endpointMediaCopyCapacity: number; - private readonly mediaRouteControllers = new Map< - string, - MediaRouteController - >(); - private readonly pendingRoutePreparations = new Map< - string, - PendingRoutePreparation - >(); - private readonly failedParentPeerIdsByViewer = new Map< - string, - Set - >(); - private readonly viewerRouteIntentsByRoom = new Map< - string, - Map - >(); - private readonly consumedSfuRefreshesByRoom = new Map>(); - private readonly selectedPeerEdgeLeasesByRoom = new Map< - string, - Map - >(); - private readonly selectedSfuIngressAttempts = new Map< - string, - SelectedSfuIngressAttempt - >(); - private readonly activeSelectedSfuIngresses = new Map< - string, - ActiveSelectedSfuIngress - >(); - private readonly relayCapacitySessionsByRoom = new Map< - string, - Map - >(); - private readonly sfuDisabledRoomIds = new Set(); - private readonly viewerQualityEvidenceStates = new Map< - string, - ViewerQualityEvidenceState - >(); - private readonly heldQualityParentExclusionsByViewer = new Map< - string, - { parentPeerId: string; parentSessionId: string; expiresAtMs: number } - >(); - private readonly roomQualityMigrationCooldownUntilMs = new Map< - string, - number - >(); - private readonly deferredHealthySfuReselections = new Map< - string, - DeferredHealthySfuReselection - >(); + private readonly rooms = new Map(); + private readonly resourceWaiterRoomIds = new Set(); private readonly sfuDrainTasks = new Map(); private readonly hostOfflineChecks = new Map(); + private readonly now: () => number; private closing = false; + constructor(private readonly options: HybridMediaRouterOptions) { assertEndpointMediaCopyCapacity(options.endpointMediaCopyCapacity); - this.peerRelayTopology = new PeerRelayTopology( - options.endpointMediaCopyCapacity, - ); - this.endpointMediaCopyCapacity = options.endpointMediaCopyCapacity; - const fallback = options.sfuFallback; - if (options.selectedEdgeTurn && !fallback) { + this.now = options.now ?? Date.now; + if (options.selectedEdgeTurn && !options.sfuFallback) { throw new Error("Selected-edge TURN requires SFU fallback"); } - if (!fallback) { - return; - } - if ( - fallback.prepareTimeoutMs !== undefined && - (!Number.isSafeInteger(fallback.prepareTimeoutMs) || - fallback.prepareTimeoutMs <= 0) - ) { - throw new Error("SFU fallback timeout is invalid"); - } - for (const [value, name] of [ - [fallback.drainRetryMs, "SFU drain retry"], - [fallback.hostOfflineCheckMs, "SFU Host-offline check"], - ] as const) { - if ( - value !== undefined && - (!Number.isSafeInteger(value) || value <= 0) - ) { - throw new Error(`${name} interval is invalid`); - } - } - new URL(fallback.url); + if (options.sfuFallback) new URL(options.sfuFallback.url); } async close(): Promise { this.closing = true; - for (const pending of this.pendingRoutePreparations.values()) { - clearTimeout(pending.timer); - } - this.pendingRoutePreparations.clear(); - for (const roomLeases of this.selectedPeerEdgeLeasesByRoom.values()) { - for (const lease of roomLeases.values()) { - clearTimeout(lease.timer); - this.releaseTurnAllocation(lease.allocationFence); - } - } - this.selectedPeerEdgeLeasesByRoom.clear(); - for (const attempt of this.selectedSfuIngressAttempts.values()) { - clearTimeout(attempt.timer); - this.releaseTurnAllocation(attempt.allocationFence); - } - this.selectedSfuIngressAttempts.clear(); - for (const deferred of this.deferredHealthySfuReselections.values()) { - if (deferred.timer) { - clearTimeout(deferred.timer); - } - } - this.deferredHealthySfuReselections.clear(); - for (const check of this.hostOfflineChecks.values()) { - clearTimeout(check.timer); - } + for (const roomId of [...this.rooms.keys()]) this.clearRoom(roomId); + this.resourceWaiterRoomIds.clear(); + for (const check of this.hostOfflineChecks.values()) clearTimeout(check.timer); this.hostOfflineChecks.clear(); - for (const ingress of this.activeSelectedSfuIngresses.values()) { - this.releaseTurnAllocation(ingress.allocationFence); - } - this.activeSelectedSfuIngresses.clear(); const turnAdmission = this.options.selectedEdgeTurn?.admission; if (turnAdmission) { for (const fence of turnAdmission.beginDrainAll()) { turnAdmission.completeDrain(fence); } - if (turnAdmission.usage().allocations !== 0) { - throw new Error("TURN allocations were not fully released during shutdown"); - } } - this.mediaRouteControllers.clear(); - this.failedParentPeerIdsByViewer.clear(); - this.viewerRouteIntentsByRoom.clear(); - this.consumedSfuRefreshesByRoom.clear(); - this.relayCapacitySessionsByRoom.clear(); - this.sfuDisabledRoomIds.clear(); - this.viewerQualityEvidenceStates.clear(); - this.heldQualityParentExclusionsByViewer.clear(); - this.roomQualityMigrationCooldownUntilMs.clear(); const fallback = this.options.sfuFallback; - if (!fallback) { - return; - } - for (const fence of fallback.admission.beginDrainAll()) { - this.scheduleSfuDrain(fence); - } - const results = await Promise.allSettled( - [...this.sfuDrainTasks].map(async ([key, task]) => { - if (task.retryTimer) { - clearTimeout(task.retryTimer); - task.retryTimer = undefined; - } - if (task.operation) { - await task.operation; - } - if (this.sfuDrainTasks.get(key) === task) { - await this.runSfuDrain(key, task); - } - }), - ); - const failures = results - .filter((result): result is PromiseRejectedResult => result.status === "rejected") - .map((result) => result.reason); - const usage = fallback.admission.usage(); - if (failures.length > 0 || usage.ingress !== 0 || usage.egress !== 0) { - throw new AggregateError( - failures, - "LiveKit rooms were not fully drained during shutdown", - ); + if (!fallback) return; + for (const fence of fallback.admission.beginDrainAll()) this.scheduleSfuDrain(fence); + for (const [key, task] of [...this.sfuDrainTasks]) { + if (task.retryTimer) clearTimeout(task.retryTimer); + if (task.operation) await task.operation; + if (this.sfuDrainTasks.get(key) === task) await this.runSfuDrain(key, task); } } connectParticipant(input: AuthenticatedRouteParticipant): HybridAuthenticationState { - // A selected transport is bound to both authenticated sessions. A client - // may replace its WebSocket without changing the stable peer id or route. - this.clearSelectedEdgeTurnsForParticipant( - input.roomId, - input.peerId, - ); - this.clearRoomViewerQualityEvidenceStates(input.roomId); - this.clearHeldQualityParentExclusion(input.roomId, input.peerId); - this.clearSfuRefreshesForPeer(input.roomId, input.peerId); + const room = this.room(input.roomId); if (input.role === "host") { + room.hostPeerId = input.peerId; this.cancelHostOfflineCheck(input.roomId); - } - if (input.role === "viewer") { - this.clearDeferredHealthySfuReselectionForParticipant( - input.roomId, + if (!room.controller) this.createController(input.roomId, room, input); + } else { + room.advertisedCapacityByViewer.set( input.peerId, + room.advertisedCapacityByViewer.get(input.peerId) ?? 0, ); - const connectionKey = viewerConnectionKey(input.roomId, input.peerId); - this.failedParentPeerIdsByViewer.delete( - connectionKey, - ); - this.viewerRouteIntentsByRoom.get(input.roomId)?.delete(input.peerId); - this.relayCapacitySessionsByRoom.get(input.roomId)?.delete(input.peerId); } - this.abortPendingRouteForTopologyChange(input.roomId, input.peerId); - const assignmentChanges = - input.role === "host" - ? this.peerRelayTopology.setHost( - input.roomId, - input.peerId, - this.connectedPeerIds(input.roomId), - ) - : this.peerRelayTopology.addViewer( - input.roomId, - input.peerId, - this.connectedPeerIds(input.roomId), - ); - this.clearChangedConnectionIds(input.roomId, assignmentChanges); - const controller = this.reconcileMediaRoute(input.roomId, input.peerId); - const activeRoute = controller?.getActiveRoute(); - const routeAssignment = - activeRoute?.assignments.get(input.peerId) ?? - participantRouteFromMediaAssignment( - this.peerRelayTopology.getAssignment(input.roomId, input.peerId), + if (room.controller) { + this.releaseResources( + room.controller.upsertParticipant({ + peerId: input.peerId, + role: input.role, + sessionId: input.sessionId, + effectiveDownstreamCapacity: + input.role === "host" + ? this.options.endpointMediaCopyCapacity + : (room.advertisedCapacityByViewer.get(input.peerId) ?? 0), + }), ); - if (!routeAssignment) { - throw new Error("Peer relay topology has no participant assignment"); + } + const snapshot = room.controller?.snapshot(); + const assignment = snapshot + ? this.assignments(input.roomId, snapshot).get(input.peerId) ?? emptyAssignment() + : emptyAssignment(); + if (input.role === "viewer") { + const edge = snapshot?.upstreamByViewer.get(input.peerId); + if (snapshot && edge && this.pathIsPhysical(input.roomId, snapshot, input.peerId)) { + this.options.setConnectionId(input.roomId, input.peerId, edge.connectionId); + } else { + this.options.deleteConnectionId(input.roomId, input.peerId); + } } return { - routeRevision: activeRoute?.revision ?? 0, - routeAssignment, - mediaAssignment: mediaAssignmentFromParticipantRoute(routeAssignment), - assignmentChanges, - activeRoute, + routeRevision: snapshot?.revision ?? 0, + routeAssignment: assignment, + mediaAssignment: mediaAssignment(assignment), }; } completeAuthentication( participant: AuthenticatedRouteParticipant, - state: HybridAuthenticationState, + _state: HybridAuthenticationState, ): void { - const activeParentPeerId = - state.routeAssignment.upstream.kind === "peer" - ? state.routeAssignment.upstream.peerId - : undefined; - this.sendMediaAssignmentChanges( - participant.roomId, - state.assignmentChanges, - participant.peerId, - ); - if ( - participant.role === "viewer" && - activeParentPeerId && - !state.assignmentChanges.some( - ({ peerId }) => peerId === activeParentPeerId, - ) - ) { - this.sendCurrentParentAssignment(participant.roomId, participant.peerId); - } - if (state.activeRoute?.sfu.publicationGeneration) { - void this.sendFreshSfuConfig( - participant.roomId, - participant.peerId, - participant.sessionId, - state.activeRoute.revision, - ); - } - this.drainViewerRouteIntents(participant.roomId); + const room = this.rooms.get(participant.roomId); + if (!room?.controller) return; + this.broadcastActive(participant.roomId, room); + void this.sendFreshSfuConfig(participant).catch(() => undefined); + this.requestPump(participant.roomId); + } + + setPaused(roomId: string, paused: boolean): void { + const room = this.rooms.get(roomId); + if (!room?.controller) return; + const before = room.controller.snapshot().revision; + this.releaseResources(room.controller.setPaused(paused)); + if (room.controller.snapshot().revision !== before) this.broadcastActive(roomId, room); + if (!paused) this.requestPump(roomId); } isActivePeerParentOf( @@ -534,9 +275,47 @@ export class HybridMediaRouter { parentPeerId: string, childPeerId: string, ): boolean { - return ( - this.resolveActivePeerEdge(roomId, childPeerId)?.parentPeerId === - parentPeerId + return this.resolveActivePeerEdge(roomId, childPeerId)?.parentPeerId === parentPeerId; + } + + resolveActivePeerEdge( + roomId: string, + childPeerId: string, + ): { revision: number; parentPeerId: string } | undefined { + const snapshot = this.rooms.get(roomId)?.controller?.snapshot(); + const edge = snapshot?.upstreamByViewer.get(childPeerId); + if ( + !snapshot || + edge?.kind !== "peer" || + !this.pathIsPhysical(roomId, snapshot, childPeerId) + ) { + return undefined; + } + return { revision: snapshot.revision, parentPeerId: edge.parentPeerId }; + } + + getViewerRouteUpstream(roomId: string, viewerPeerId: string): MediaRouteUpstream { + const snapshot = this.rooms.get(roomId)?.controller?.snapshot(); + return snapshot + ? (this.assignments(roomId, snapshot).get(viewerPeerId)?.upstream ?? { kind: "none" }) + : { kind: "none" }; + } + + viewerSfuMediaStateIsCurrent( + roomId: string, + viewerPeerId: string, + revision: number, + publicationGeneration: string, + ): boolean { + const snapshot = this.rooms.get(roomId)?.controller?.snapshot(); + const edge = snapshot?.upstreamByViewer.get(viewerPeerId); + return Boolean( + snapshot?.revision === revision && + edge?.kind === "sfu" && + edge.physicalActive && + edge.publicationGeneration === publicationGeneration && + snapshot.hostPublication?.physicalActive && + snapshot.hostPublication.generation === publicationGeneration, ); } @@ -550,616 +329,92 @@ export class HybridMediaRouter { signalKind: "candidate" | "description"; descriptionType?: "offer" | "answer"; }): boolean | "probe" | undefined { - const lease = this.selectedPeerEdgeLeaseForPeers( - input.roomId, - input.sourcePeerId, - input.targetPeerId, - ); - if (!lease) { - const pending = this.pendingRoutePreparations.get(input.roomId); - const qualityProbe = pending?.peerQualityReselection; - const viewerPeerId = pending?.intentPeerId; - const parentToViewer = - qualityProbe?.newParentPeerId === input.sourcePeerId && - viewerPeerId === input.targetPeerId; - const viewerToParent = - viewerPeerId === input.sourcePeerId && - qualityProbe?.newParentPeerId === input.targetPeerId; - if (pending && qualityProbe && (parentToViewer || viewerToParent)) { - if (!this.peerQualityReselectionIsCurrent(input.roomId, pending)) { - return false; - } - if ( - parentToViewer && - input.signalKind === "description" && - input.descriptionType === "offer" - ) { - qualityProbe.newConnectionId ??= input.connectionId; - } - return ( - qualityProbe.newConnectionId === input.connectionId && + const controller = this.rooms.get(input.roomId)?.controller; + const snapshot = controller?.snapshot(); + if (!controller || !snapshot) return undefined; + const operation = snapshot.operation; + const current = operation?.current; + if (operation && current?.tuple.kind === "peer") { + const parent = this.connectedPeer(input.roomId, current.tuple.parentPeerId); + const child = this.options.roomStore.getConnectedViewer( + input.roomId, + operation.childPeerId, + ); + const parentToChild = + input.sourcePeerId === current.tuple.parentPeerId && + input.targetPeerId === operation.childPeerId; + const childToParent = + input.sourcePeerId === operation.childPeerId && + input.targetPeerId === current.tuple.parentPeerId; + return Boolean( + parent && + child && + parent.sessionId === (parentToChild ? input.sourceSessionId : input.targetSessionId) && + child.sessionId === (childToParent ? input.sourceSessionId : input.targetSessionId) && + input.connectionId === current.connectionId && (input.signalKind === "candidate" || - (parentToViewer && input.descriptionType === "offer") || - (viewerToParent && input.descriptionType === "answer")) - ) ? "probe" : false; - } - const probe = pending?.healthyReselection; - const parentToRoot = probe?.parentPeerId === input.sourcePeerId && - probe.rootPeerId === input.targetPeerId; - const rootToParent = probe?.rootPeerId === input.sourcePeerId && - probe.parentPeerId === input.targetPeerId; - if (!pending || !probe || (!parentToRoot && !rootToParent)) { - return undefined; - } - if (!this.healthySfuReselectionIsCurrent(input.roomId, pending)) { - return false; - } - const connectionId = this.options.getConnectionId(input.roomId, probe.rootPeerId); - return parentToRoot && input.descriptionType === "offer" - ? connectionId === undefined - : connectionId === input.connectionId && - (input.signalKind === "candidate" || - (rootToParent && input.descriptionType === "answer")); + (parentToChild && input.descriptionType === "offer") || + (childToParent && input.descriptionType === "answer")), + ) + ? "probe" + : false; } - const parentToViewer = input.sourcePeerId === lease.parentPeerId; - const currentRoute = this.mediaRouteControllers - .get(input.roomId) - ?.getActiveRoute(); - return Boolean( - currentRoute && - input.connectionId === lease.newConnectionId && - currentRoute.revision === lease.currentRouteRevision && - this.selectedPeerEdgeLeaseIsCurrent(input.roomId, lease, currentRoute) && - ((parentToViewer && - input.sourceSessionId === lease.parentSessionId && - input.targetSessionId === lease.viewerSessionId) || - (!parentToViewer && - input.sourceSessionId === lease.viewerSessionId && - input.targetSessionId === lease.parentSessionId)) && - (input.signalKind === "candidate" || - (!lease.answered && - ((parentToViewer && input.descriptionType === "offer") || - (!parentToViewer && input.descriptionType === "answer")))) - ); + + const childPeerId = this.childForPair(snapshot, input.sourcePeerId, input.targetPeerId); + const edge = childPeerId ? snapshot.upstreamByViewer.get(childPeerId) : undefined; + if (!childPeerId || edge?.kind !== "peer" || !edge.physicalActive) return undefined; + const parentToChild = input.sourcePeerId === edge.parentPeerId; + const sessionsMatch = + (parentToChild && + input.sourceSessionId === edge.parentSessionId && + input.targetSessionId === edge.childSessionId) || + (!parentToChild && + input.sourceSessionId === edge.childSessionId && + input.targetSessionId === edge.parentSessionId); + if (!sessionsMatch) return false; + if ( + edge.transport === "direct" && + parentToChild && + input.signalKind === "description" && + input.descriptionType === "offer" && + input.connectionId !== edge.connectionId + ) { + const adopted = controller.adoptDirectConnection({ + childPeerId, + childSessionId: edge.childSessionId, + parentSessionId: edge.parentSessionId, + routeRevision: snapshot.revision, + connectionId: edge.connectionId, + newConnectionId: input.connectionId, + }); + if (adopted) this.options.setConnectionId(input.roomId, childPeerId, input.connectionId); + return adopted; + } + return input.connectionId === edge.connectionId; } markSelectedEdgeTurnAnswered( - roomId: string, - viewerPeerId: string, - connectionId: string, + _roomId: string, + _viewerPeerId: string, + _connectionId: string, ): void { - const lease = this.selectedPeerEdgeLease(roomId, viewerPeerId); - if ( - lease?.viewerPeerId === viewerPeerId && - lease.newConnectionId === connectionId - ) { - if (!this.options.selectedEdgeTurn?.admission.commit(lease.allocationFence)) { - this.settleSelectedPeerEdgeLease(roomId, viewerPeerId); - this.sendError( - lease.viewerSessionId, - "SERVER_ERROR", - "Selected relay edge lost its resource reservation", - ); - return; - } - lease.answered = true; - clearTimeout(lease.timer); - this.drainViewerRouteIntents(roomId); - } - } - - resolveActivePeerEdge( - roomId: string, - childPeerId: string, - ): { revision: number; parentPeerId: string } | undefined { - const controller = this.mediaRouteControllers.get(roomId); - if (!controller) { - return undefined; - } - const active = controller.getActiveRoute(); - const assignments = active.assignments; - const child = assignments.get(childPeerId); - if ( - child?.upstream.kind !== "peer" || - assignments - .get(child.upstream.peerId) - ?.childPeerIds.includes(childPeerId) !== true - ) { - return undefined; - } - return { - revision: active.revision, - parentPeerId: child.upstream.peerId, - }; - } - - getViewerRouteUpstream( - roomId: string, - viewerPeerId: string, - ): MediaRouteUpstream { - const assignment = this.mediaRouteControllers - .get(roomId) - ?.getActiveRoute() - .assignments.get(viewerPeerId); - if (!assignment || assignment.upstream.kind === "none") { - return { kind: "none" }; - } - if (assignment.upstream.kind === "peer") { - return { kind: "peer", peerId: assignment.upstream.peerId }; - } - return { kind: "sfu" }; - } - - viewerSfuMediaStateIsCurrent( - roomId: string, - viewerPeerId: string, - revision: number, - sfuPublicationGeneration: string, - ): boolean { - const active = this.mediaRouteControllers.get(roomId)?.getActiveRoute(); - return Boolean( - active?.revision === revision && - active.assignments.get(viewerPeerId)?.upstream.kind === "sfu" && - active.sfu.publicationGeneration === sfuPublicationGeneration, - ); - } - - handleViewerQualityEvidence(input: ForwardedViewerQualityEvidence): void { - const { evidence, roomId, viewerSessionId, parentSessionId } = input; - const { viewerPeerId, parentPeerId } = evidence; - const viewer = this.options.roomStore.getConnectedViewer( - roomId, - viewerPeerId, - ); - const parent = this.connectedPeer(roomId, parentPeerId); - const edge = this.resolveActivePeerEdge(roomId, viewerPeerId); - if ( - viewer?.sessionId !== viewerSessionId || - parent?.sessionId !== parentSessionId || - edge?.parentPeerId !== parentPeerId || - edge.revision !== evidence.guard.routeRevision || - this.options.getConnectionId(roomId, viewerPeerId) !== - evidence.guard.connectionId - ) { - return; - } - - const connectionKey = viewerConnectionKey(roomId, viewerPeerId); - const existingIntent = this.viewerRouteIntentsByRoom - .get(roomId) - ?.get(viewerPeerId); - if ( - existingIntent?.qualityGuard && - !this.qualityIntentIsCurrent(roomId, viewerPeerId, existingIntent) - ) { - this.discardQualityIntent(roomId, viewerPeerId, existingIntent); - } - const now = this.options.now?.() ?? Date.now(); - if (!Number.isFinite(now)) { - return; - } - const previous = this.viewerQualityEvidenceStates.get(connectionKey); - const sameIdentity = - previous?.viewerSessionId === viewerSessionId && - previous.parentPeerId === parentPeerId && - previous.parentSessionId === parentSessionId && - previous.connectionId === evidence.guard.connectionId && - previous.routeRevision === evidence.guard.routeRevision; - const canContinueStreak = - sameIdentity && - previous !== undefined && - previous.pendingViewerEvidence === null && - previous.lastCorrelatedAtMs !== null && - now - previous.lastCorrelatedAtMs <= VIEWER_QUALITY_EVIDENCE_GAP_MS; - this.viewerQualityEvidenceStates.set(connectionKey, { - viewerSessionId, - parentPeerId, - parentSessionId, - connectionId: evidence.guard.connectionId, - routeRevision: evidence.guard.routeRevision, - lastCorrelatedAtMs: canContinueStreak - ? previous.lastCorrelatedAtMs - : null, - badWindowCount: canContinueStreak ? previous.badWindowCount : 0, - badWindowKind: canContinueStreak ? previous.badWindowKind : null, - lastCorrelatedFps: sameIdentity - ? previous?.lastCorrelatedFps ?? null - : null, - pendingViewerEvidence: { acceptedAtMs: now, evidence }, - }); - } - - handleParentEdgeQualityEvidence( - input: ForwardedParentEdgeQualityEvidence, - ): void { - const { evidence, roomId, viewerSessionId, parentSessionId } = input; - const viewerPeerId = evidence.viewerPeerId; - const viewer = this.options.roomStore.getConnectedViewer( - roomId, - viewerPeerId, - ); - const edge = this.resolveActivePeerEdge(roomId, viewerPeerId); - const parent = edge - ? this.connectedPeer(roomId, edge.parentPeerId) - : undefined; - if ( - viewer?.sessionId !== viewerSessionId || - parent?.sessionId !== parentSessionId || - edge?.revision !== evidence.guard.routeRevision || - this.options.getConnectionId(roomId, viewerPeerId) !== - evidence.guard.connectionId - ) { - return; - } - - const connectionKey = viewerConnectionKey(roomId, viewerPeerId); - const state = this.viewerQualityEvidenceStates.get(connectionKey); - const pending = state?.pendingViewerEvidence; - if ( - !state || - !pending || - state.viewerSessionId !== viewerSessionId || - state.parentPeerId !== edge.parentPeerId || - state.parentSessionId !== parentSessionId || - state.connectionId !== evidence.guard.connectionId || - state.routeRevision !== evidence.guard.routeRevision || - pending.evidence.sequence !== evidence.viewerSequence - ) { - return; - } - state.pendingViewerEvidence = null; - - const now = this.options.now?.() ?? Date.now(); - if ( - !Number.isFinite(now) || - now < pending.acceptedAtMs || - now - pending.acceptedAtMs > VIEWER_QUALITY_EVIDENCE_GAP_MS - ) { - state.badWindowCount = 0; - state.badWindowKind = null; - state.lastCorrelatedAtMs = null; - return; - } - state.lastCorrelatedFps = { - correlatedAtMs: now, - framesPerSecond: pending.evidence.metrics.framesPerSecond, - }; - const existingIntent = this.viewerRouteIntentsByRoom - .get(roomId) - ?.get(viewerPeerId); - this.pruneHeldQualityParentExclusion(roomId, viewerPeerId, now); - const failedParents = this.failedParentPeerIdsByViewer.get(connectionKey); - if (existingIntent || failedParents?.has(edge.parentPeerId)) { - state.badWindowCount = 0; - state.badWindowKind = null; - state.lastCorrelatedAtMs = now; - return; - } - - const hardParentProof = isHardBadParentEdgeQualityProof(evidence.proof); - const badWindowKind = !hardParentProof - ? null - : isHardBadViewerQualityWindow(pending.evidence) - ? "severe" - : this.isHardBadRelativeRelayFpsWindow( - roomId, - edge.parentPeerId, - parentSessionId, - pending.evidence, - now, - ) - ? "relative-fps" - : null; - state.badWindowCount = badWindowKind - ? state.badWindowKind === badWindowKind && - state.lastCorrelatedAtMs !== null && - now - state.lastCorrelatedAtMs <= VIEWER_QUALITY_EVIDENCE_GAP_MS - ? state.badWindowCount + 1 - : 1 - : 0; - state.badWindowKind = badWindowKind; - state.lastCorrelatedAtMs = now; - if (state.badWindowCount < VIEWER_QUALITY_BAD_WINDOWS_TO_REASSIGN) { - return; - } - - const cooldownUntil = this.roomQualityMigrationCooldownUntilMs.get(roomId); - if (cooldownUntil !== undefined && now < cooldownUntil) { - state.badWindowCount = 0; - state.badWindowKind = null; - return; - } else if (cooldownUntil !== undefined) { - this.roomQualityMigrationCooldownUntilMs.delete(roomId); - } - - let roomIntents = this.viewerRouteIntentsByRoom.get(roomId); - if (!roomIntents) { - roomIntents = new Map(); - this.viewerRouteIntentsByRoom.set(roomId, roomIntents); - } - let excludedParents = this.failedParentPeerIdsByViewer.get(connectionKey); - if (!excludedParents) { - excludedParents = new Set(); - this.failedParentPeerIdsByViewer.set(connectionKey, excludedParents); - } - excludedParents.add(edge.parentPeerId); - roomIntents.set(viewerPeerId, { - failedParentPeerId: edge.parentPeerId, - failedParentSessionId: parentSessionId, - failedConnectionId: evidence.guard.connectionId, - sessionId: viewerSessionId, - sfuAttempts: 0, - unavailableReported: false, - qualityGuard: { - kind: badWindowKind!, - viewerSessionId, - connectionId: evidence.guard.connectionId, - routeRevision: evidence.guard.routeRevision, - parentPeerId: edge.parentPeerId, - parentSessionId, - exclusionOwnedByQuality: true, - }, - }); - this.viewerQualityEvidenceStates.delete(connectionKey); - this.drainViewerRouteIntents(roomId); - } - - private isHardBadRelativeRelayFpsWindow( - roomId: string, - parentPeerId: string, - parentSessionId: string, - childEvidence: Extract< - ServerMessage, - { type: "viewer-quality-evidence" } - >, - now: number, - ): boolean { - const parentState = this.viewerQualityEvidenceStates.get( - viewerConnectionKey(roomId, parentPeerId), - ); - const correlated = parentState?.lastCorrelatedFps; - const parentEdge = this.resolveActivePeerEdge(roomId, parentPeerId); - if ( - this.options.roomStore.getConnectedViewer(roomId, parentPeerId) - ?.sessionId !== parentSessionId || - !parentState || - !correlated || - parentState.viewerSessionId !== parentSessionId || - parentEdge?.parentPeerId !== parentState.parentPeerId || - parentEdge.revision !== parentState.routeRevision || - this.options.getConnectionId(roomId, parentPeerId) !== - parentState.connectionId || - now < correlated.correlatedAtMs || - now - correlated.correlatedAtMs > VIEWER_QUALITY_EVIDENCE_GAP_MS - ) { - return false; - } - const childFps = childEvidence.metrics.framesPerSecond; - const parentInboundFps = correlated.framesPerSecond; - return ( - childFps !== null && - parentInboundFps !== null && - parentInboundFps > 0 && - childFps < parentInboundFps * VIEWER_QUALITY_RELAY_FPS_RATIO - ); + // Resource admission commits only with exact child media proof. } setViewerRelayCapacity( participant: AuthenticatedRouteParticipant, - downstreamEdges: RelayDownstreamEdges, + downstreamEdges: number, ): void { - if (participant.role !== "viewer") { - return; - } - let advertised = this.relayCapacitySessionsByRoom.get(participant.roomId); - if (!advertised) { - advertised = new Map(); - this.relayCapacitySessionsByRoom.set(participant.roomId, advertised); - } - advertised.set(participant.peerId, participant.sessionId); - const controller = this.mediaRouteControllers.get(participant.roomId); - const active = controller?.getActiveRoute(); - const routeIntent = this.viewerRouteIntentsByRoom - .get(participant.roomId) - ?.get(participant.peerId); - const allowAdmissionRescue = - downstreamEdges > 0 && - active?.sfu.publicationGeneration === null && - active.assignments.get(participant.peerId)?.upstream.kind === "none" && - controller?.getPendingRoute() === undefined && - !this.pendingRoutePreparations.has(participant.roomId) && - (routeIntent === undefined || routeIntent.failedParentPeerId === null) && - !this.failedParentPeerIdsByViewer.get( - viewerConnectionKey(participant.roomId, participant.peerId), - )?.size; - const changes = this.peerRelayTopology.setViewerRelayCapacity( - participant.roomId, - participant.peerId, - downstreamEdges, - this.connectedPeerIds(participant.roomId), - { rescueUnassignedRelay: allowAdmissionRescue }, - ); - if (changes.length > 0) { - this.clearChangedConnectionIds(participant.roomId, changes); - this.reconcileMediaRoute(participant.roomId); - this.sendMediaAssignmentChanges(participant.roomId, changes); - } else if (active) { - this.reconcileSelectedPeerEdgeLeases(participant.roomId, active); - } - const assignment = this.mediaRouteControllers - .get(participant.roomId) - ?.getActiveRoute() - .assignments.get(participant.peerId); - let roomIntents = this.viewerRouteIntentsByRoom.get(participant.roomId); - if (assignment?.upstream.kind === "none") { - if (!roomIntents) { - roomIntents = new Map(); - this.viewerRouteIntentsByRoom.set(participant.roomId, roomIntents); - } - const existing = roomIntents.get(participant.peerId); - if ( - existing?.sessionId !== participant.sessionId || - existing.failedParentPeerId !== null - ) { - roomIntents.set(participant.peerId, { - failedParentPeerId: null, - sessionId: participant.sessionId, - sfuAttempts: 0, - unavailableReported: false, - }); - } - } else if ( - roomIntents?.get(participant.peerId)?.failedParentPeerId === null - ) { - roomIntents.delete(participant.peerId); - } - this.drainViewerRouteIntents(participant.roomId); - } - - handleHealthySfuReselection( - participant: AuthenticatedRouteParticipant, - revision: number, - ): void { - const roomId = participant.roomId; - const controller = this.mediaRouteControllers.get(roomId); - const active = controller?.getActiveRoute(); - const publicationGeneration = active?.sfu.publicationGeneration; - const shareGeneration = this.options.getShareGeneration(roomId); + const room = this.room(participant.roomId); + room.advertisedCapacityByViewer.set(participant.peerId, downstreamEdges); if ( - participant.role !== "viewer" || - !controller || - controller.getPendingRoute() || - this.roomPeerMigrationAttempt(roomId, participant.peerId) !== null || - active?.revision !== revision || - active.sfu.rootPeerIds.length !== 1 || - active.sfu.rootPeerIds[0] !== participant.peerId || - this.options.roomStore.getConnectedViewer(roomId, participant.peerId) - ?.sessionId !== participant.sessionId || - !publicationGeneration || - !shareGeneration + room.controller?.setEffectiveCapacity( + participant.peerId, + participant.sessionId, + downstreamEdges, + ) ) { - return; - } - const now = this.options.now?.() ?? Date.now(); - const cooldownUntil = - this.roomQualityMigrationCooldownUntilMs.get(roomId) ?? 0; - if (!Number.isFinite(now)) { - return; - } - if (cooldownUntil > now) { - this.deferHealthySfuReselection( - participant, - revision, - shareGeneration, - publicationGeneration, - cooldownUntil, - now, - ); - return; - } - this.clearDeferredHealthySfuReselection(roomId); - this.startRoomQualityMigrationCooldown(roomId); - const hostPeerId = this.peerRelayTopology.getHostPeerId(roomId); - const parentPeerId = this.peerRelayTopology.getAssignment( - roomId, - participant.peerId, - )?.parentPeerId; - if (!hostPeerId || !parentPeerId) { - return; - } - const host = this.connectedPeer(roomId, hostPeerId); - const parent = this.connectedPeer(roomId, parentPeerId); - const activeParent = active.assignments.get(parentPeerId); - const parentSupportsOutboundRelay = - parentPeerId === hostPeerId - ? activeParent?.upstream.kind === "none" - : activeParent?.upstream.kind === "peer"; - const capacity = this.peerRelayTopology.getDownstreamCapacity( - roomId, - parentPeerId, - ); - if ( - !host || - !parent || - !activeParent || - !parentSupportsOutboundRelay || - !endpointMediaCopyCountFits( - this.activeParentMediaEdges(roomId, parentPeerId, activeParent) + 1, - capacity, - "transition", - ) || - (parentPeerId !== hostPeerId && - this.relayCapacitySessionsByRoom.get(roomId)?.get(parentPeerId) !== - parent.sessionId) - ) { - return; - } - const assignments = participantRoutesFromTopology( - this.peerRelayTopology.getAssignments(roomId), - hostPeerId, - null, - [], - this.endpointMediaCopyCapacity, - ); - const rootAssignment = assignments.get(participant.peerId)!; - const parentAssignment = assignments.get(parentPeerId)!; - let pendingRevision: number | undefined; - try { - pendingRevision = controller.prepare({ - assignments, - expectedParticipantIds: new Set([participant.peerId]), - }); - } catch { - return; - } - if (pendingRevision === undefined) { - return; - } - const timer = setTimeout( - () => this.abortPendingRoute(roomId), - this.options.sfuFallback?.prepareTimeoutMs ?? - DEFAULT_SFU_PREPARE_TIMEOUT_MS, - ); - timer.unref(); - this.options.deleteConnectionId(roomId, participant.peerId); - this.pendingRoutePreparations.set(roomId, { - revision: pendingRevision, - intentPeerId: participant.peerId, - routeIntent: { - failedParentPeerId: null, - sessionId: participant.sessionId, - sfuAttempts: 0, - unavailableReported: false, - }, - qualityIntent: null, - expectedSessionIds: new Map([ - [participant.peerId, participant.sessionId], - [parentPeerId, parent.sessionId], - [hostPeerId, host.sessionId], - ]), - grantsIssued: true, - hostSfuIngressTurnAttempted: false, - healthyReselection: { - activeRevision: revision, - shareGeneration, - publicationGeneration, - rootPeerId: participant.peerId, - parentPeerId, - }, - peerQualityReselection: null, - sfuResourceFence: null, - timer, - }); - for (const [sessionId, assignment] of [ - [participant.sessionId, rootAssignment], - [parent.sessionId, parentAssignment], - ] as const) { - this.options.sendToSession(sessionId, { - type: "route-update", - revision: pendingRevision, - phase: "prepare", - assignment, - }); + this.requestPump(participant.roomId); } } @@ -1167,2941 +422,1114 @@ export class HybridMediaRouter { participant: AuthenticatedRouteParticipant, message: Extract, ): void { - const controller = this.mediaRouteControllers.get(participant.roomId); - if (!controller) { - return; - } - if (message.phase === "active") { - if (participant.role === "viewer") { - this.notifyViewerMediaState(participant, message.revision, true); - return; - } - const active = controller.getActiveRoute(); - if ( - active.revision === message.revision && - active.assignments.has(participant.peerId) - ) { - if (participant.role === "host") { - this.markSelectedSfuIngressAnswered( - participant.roomId, - participant.peerId, - participant.sessionId, - message.revision, - ); - } - return; - } - return; - } - - const pending = this.pendingRoutePreparations.get(participant.roomId); - const pendingRoute = controller.getPendingRoute(); + if (participant.role !== "viewer" || message.phase !== "prepare") return; + const room = this.rooms.get(participant.roomId); + const controller = room?.controller; + const operation = controller?.snapshot().operation; + const current = operation?.current; if ( - !pending || - !pendingRoute || - pending.revision !== message.revision || - pendingRoute.route.revision !== message.revision || - !pending.grantsIssued + !room || + !controller || + !operation || + !current || + operation.childPeerId !== participant.peerId || + operation.childSessionId !== participant.sessionId || + current.revision !== message.revision ) { return; } - const healthy = pending.healthyReselection; - if (healthy) { - const current = this.healthySfuReselectionIsCurrent(participant.roomId, pending); - if ( - participant.peerId !== healthy.rootPeerId || - !this.options.getConnectionId(participant.roomId, healthy.rootPeerId) || - !current - ) { - if (!current) { - this.abortPendingRoute(participant.roomId); - } - return; - } - } - const qualityProbe = pending.peerQualityReselection; - if (qualityProbe) { - const current = this.peerQualityReselectionIsCurrent( - participant.roomId, - pending, - ); - if ( - participant.role !== "viewer" || - participant.peerId !== pending.intentPeerId || - qualityProbe.newConnectionId === null || - !current - ) { - if (!current) { - this.abortPendingRoute(participant.roomId); - } - return; - } - } - const expectedSessionId = pending.expectedSessionIds.get(participant.peerId); - if (!expectedSessionId) { - this.sendError( - participant.sessionId, - "FORBIDDEN", - "Participant is not part of this route plan", - ); - return; - } - if (participant.sessionId !== expectedSessionId) { - return; - } - - if (participant.role === "host") { - this.markSelectedSfuIngressAnswered( + const before = controller.snapshot().revision; + const settled = controller.candidateReady( + { + childPeerId: participant.peerId, + childSessionId: participant.sessionId, + revision: message.revision, + connectionId: current.connectionId, + }, + this.now(), + (reservation) => this.commitReservation(reservation), + ); + this.releaseResources(settled.released); + if (settled.accepted) { + this.resourceWaiterRoomIds.delete(participant.roomId); + this.options.setConnectionId( participant.roomId, participant.peerId, - participant.sessionId, - message.revision, + current.connectionId, ); } - - controller.ready(participant.peerId, message.revision, "prepare"); - this.commitPendingRoute(participant.roomId, message.revision); + if (controller.snapshot().revision !== before) this.broadcastActive(participant.roomId, room); + this.requestPump(participant.roomId); } handleRouteMediaUnavailable( participant: AuthenticatedRouteParticipant, message: Extract, ): void { - this.notifyViewerMediaState(participant, message.revision, false); - } - - private notifyViewerMediaState( - participant: AuthenticatedRouteParticipant, - revision: number, - ready: boolean, - ): void { + if (participant.role !== "viewer") return; + const room = this.rooms.get(participant.roomId); + const controller = room?.controller; + const snapshot = controller?.snapshot(); + const edge = snapshot?.upstreamByViewer.get(participant.peerId); if ( - participant.role !== "viewer" || - this.connectedPeer(participant.roomId, participant.peerId)?.sessionId !== - participant.sessionId + !room || + !controller || + !snapshot || + snapshot.paused || + snapshot.revision !== message.revision || + edge?.kind !== "sfu" || + edge.childSessionId !== participant.sessionId ) { return; } - const active = this.mediaRouteControllers - .get(participant.roomId) - ?.getActiveRoute(); - const assignment = active?.assignments.get(participant.peerId); - if ( - !active || - active.revision !== revision || - assignment?.upstream.kind !== "sfu" || - active.sfu.publicationGeneration === null - ) { - return; - } - this.options.onViewerMediaState?.( - participant.roomId, - participant.peerId, - participant.sessionId, - revision, - active.sfu.publicationGeneration, - ready, - ); + controller.invalidateEdge({ + childPeerId: participant.peerId, + childSessionId: participant.sessionId, + routeRevision: snapshot.revision, + connectionId: edge.connectionId, + }); + this.requestPump(participant.roomId); } handleRouteFailed( participant: AuthenticatedRouteParticipant, message: Extract, ): void { - const healthyPending = this.pendingRoutePreparations.get(participant.roomId); - const healthy = healthyPending?.healthyReselection; - const qualityProbe = healthyPending?.peerQualityReselection; - const qualityGuard = healthyPending?.qualityIntent?.guard; - const ownsHealthyActiveSfu = healthyPending - ? this.ownsActiveSfuForHealthyReselection(participant, healthyPending) - : false; - const ownsQualityActiveSfu = healthyPending - ? this.ownsActiveOrdinarySfuForPeerQualityReselection( - participant, - healthyPending, - ) - : false; - if ( - message.phase === "active" && - healthy?.activeRevision === message.revision && - ownsHealthyActiveSfu - ) { - this.abortPendingRoute(participant.roomId); - const rollbackRevision = this.mediaRouteControllers - .get(participant.roomId) - ?.getActiveRoute().revision; - if (rollbackRevision === undefined) { - return; - } - message = { ...message, revision: rollbackRevision }; - } - const qualityActiveIngress = this.activeSelectedSfuIngresses.get( - participant.roomId, - ); - const qualityActiveRoute = this.mediaRouteControllers - .get(participant.roomId) - ?.getActiveRoute(); - if ( - qualityProbe && - participant.role === "host" && - qualityActiveIngress?.hostPeerId === participant.peerId && - qualityActiveIngress.hostSessionId === participant.sessionId && - qualityActiveIngress.connectionId === message.connectionId && - qualityActiveIngress.shareGeneration === this.options.getShareGeneration(participant.roomId) && - qualityActiveRoute?.sfu.publicationGeneration === qualityActiveIngress.publicationGeneration && - ((message.phase === "active" && message.revision === qualityActiveRoute.revision) || - (message.phase === "prepare" && message.revision === healthyPending?.revision)) - ) { - this.abortPeerQualityReselectionForHardFailure(participant.roomId); - this.settleActiveSelectedSfuIngress(participant.roomId); - this.failBackToPeerBaseline(participant.roomId); - return; - } - if ( - message.phase === "prepare" && - message.connectionId === null && - healthyPending?.revision === message.revision && - ownsQualityActiveSfu - ) { - this.abortPeerQualityReselectionForHardFailure( - participant.roomId, - participant.role === "viewer" ? participant.peerId : undefined, - ); - const rollbackRevision = this.mediaRouteControllers - .get(participant.roomId) - ?.getActiveRoute().revision; - if (rollbackRevision === undefined) { - return; - } - message = { - ...message, - revision: rollbackRevision, - phase: "active", - }; - } - if ( - message.phase === "prepare" && - healthyPending?.revision === message.revision && - healthy - ) { - if (!ownsHealthyActiveSfu) { - return; - } - const activeIngress = this.activeSelectedSfuIngresses.get( - participant.roomId, - ); - const hasCurrentSelectedHostIngress = - participant.role === "host" && - activeIngress?.hostPeerId === participant.peerId && - activeIngress.hostSessionId === participant.sessionId && - activeIngress.shareGeneration === healthy.shareGeneration && - activeIngress.publicationGeneration === - healthy.publicationGeneration; - const activeSfuTransportFailed = hasCurrentSelectedHostIngress - ? activeIngress.connectionId === message.connectionId - : message.connectionId === null; - if (activeSfuTransportFailed) { - if (participant.role === "host") { - this.settleActiveSelectedSfuIngress(participant.roomId); - } - this.abortPendingRoute(participant.roomId, undefined, false); - return; - } - if ( - participant.role === "viewer" && - participant.peerId === healthy.rootPeerId && - message.connectionId === - this.options.getConnectionId(participant.roomId, healthy.rootPeerId) - ) { - this.abortPendingRoute(participant.roomId); - } - return; - } - if ( - message.phase === "prepare" && - healthyPending?.revision === message.revision && - qualityProbe - ) { - if ( - participant.role === "viewer" && - participant.peerId === healthyPending.intentPeerId && - participant.sessionId === qualityGuard?.viewerSessionId && - qualityProbe.newConnectionId !== null && - message.connectionId === qualityProbe.newConnectionId - ) { - this.abortPendingRoute(participant.roomId); - } - return; - } + const room = this.rooms.get(participant.roomId); + const controller = room?.controller; + const snapshot = controller?.snapshot(); + if (!room || !controller || !snapshot) return; if (message.phase === "prepare") { - this.handlePrepareRouteFailed(participant, message); - return; - } - if ( - message.connectionId !== null && - this.finishSelectedEdgeTurnFailure(participant, message) - ) { - return; - } - const controller = this.mediaRouteControllers.get(participant.roomId); - const active = controller?.getActiveRoute(); - if (!controller || !active || active.revision !== message.revision) { - return; - } - if (message.connectionId === null) { - const assignment = active.assignments.get(participant.peerId); - const hostOwnsGeneration = - participant.role === "host" && - active.sfu.publicationGeneration !== null && - assignment?.sfuPublicationGeneration === - active.sfu.publicationGeneration; - const isSfuRoot = - assignment?.upstream.kind === "sfu" && - active.sfu.rootPeerIds.includes(participant.peerId); - if (!hostOwnsGeneration && !isSfuRoot) { - this.sendError( - participant.sessionId, - "FORBIDDEN", - "Participant does not own an active SFU route", - ); - return; - } - this.abortPeerQualityReselectionForHardFailure( - participant.roomId, - participant.role === "viewer" ? participant.peerId : undefined, - ); - if ( - hostOwnsGeneration && - this.startSelectedSfuIngressTurn( - participant.roomId, - participant.peerId, - participant.sessionId, - controller.getActiveRoute(), - ) - ) { - return; - } - if ( - isSfuRoot && + const operation = snapshot.operation; + const current = operation?.current; + if (!operation || !current || current.revision !== message.revision) return; + const ownsChild = participant.role === "viewer" && - this.startSelectedEdgeTurn(participant.roomId, participant.peerId, true) + participant.peerId === operation.childPeerId && + participant.sessionId === operation.childSessionId; + const ownsParent = + current.tuple.kind === "peer" && + participant.peerId === current.tuple.parentPeerId && + this.connectedPeer(participant.roomId, participant.peerId)?.sessionId === + participant.sessionId; + const ownsPublication = + current.tuple.kind === "sfu" && + participant.role === "host" && + participant.peerId === room.hostPeerId; + if (!ownsChild && !ownsParent && !ownsPublication) return; + if (message.connectionId && message.connectionId !== current.connectionId) return; + const before = snapshot.revision; + const settled = controller.candidateFailed( + { + childPeerId: operation.childPeerId, + childSessionId: operation.childSessionId, + revision: current.revision, + connectionId: current.connectionId, + }, + this.now(), + ); + this.releaseResources(settled.released); + if (controller.snapshot().revision !== before) this.broadcastActive(participant.roomId, room); + this.requestPump(participant.roomId); + return; + } + + if (snapshot.revision !== message.revision) return; + if (participant.role === "host") { + const publication = snapshot.hostPublication; + if ( + !publication || + publication.hostSessionId !== participant.sessionId || + (message.connectionId && message.connectionId !== publication.connectionId) ) { return; } - this.failBackToPeerBaseline(participant.roomId); - return; - } - if (participant.role !== "viewer") { - this.sendError( - participant.sessionId, - "FORBIDDEN", - "Only an active viewer route may fail", + controller.invalidateHostPublication({ + hostSessionId: participant.sessionId, + routeRevision: snapshot.revision, + generation: publication.generation, + connectionId: publication.connectionId, + }); + const invalid = controller.snapshot(); + this.releaseResources( + controller.retireHostPublication({ + hostSessionId: participant.sessionId, + routeRevision: invalid.revision, + generation: publication.generation, + connectionId: publication.connectionId, + }), ); + this.broadcastActive(participant.roomId, room); + this.requestPump(participant.roomId); return; } - const assignment = active.assignments.get(participant.peerId); - if (!assignment) { - return; - } - if (assignment.upstream.kind !== "peer") { - this.sendError( - participant.sessionId, - "FORBIDDEN", - "The active route has no peer edge to recover", - ); - return; - } - const connectionKey = viewerConnectionKey( - participant.roomId, - participant.peerId, - ); - if ( - this.options.getConnectionId(participant.roomId, participant.peerId) !== - message.connectionId - ) { - return; - } - this.abortPeerQualityReselectionForHardFailure( - participant.roomId, - participant.peerId, - ); - const now = this.options.now?.() ?? Date.now(); - this.pruneHeldQualityParentExclusion( - participant.roomId, - participant.peerId, - now, - ); - const heldQualityParent = - this.heldQualityParentExclusionsByViewer.get(connectionKey); - const heldQualityFailure = - heldQualityParent?.parentPeerId === assignment.upstream.peerId; - if (heldQualityFailure) { - this.heldQualityParentExclusionsByViewer.delete(connectionKey); - } else if (heldQualityParent) { - this.clearHeldQualityParentExclusion( - participant.roomId, - participant.peerId, - ); - } - let failedParents = this.failedParentPeerIdsByViewer.get(connectionKey); - if (!failedParents) { - failedParents = new Set(); - this.failedParentPeerIdsByViewer.set(connectionKey, failedParents); - } + const edge = snapshot.upstreamByViewer.get(participant.peerId); if ( - failedParents.has(assignment.upstream.peerId) && - !heldQualityFailure + !edge || + edge.childSessionId !== participant.sessionId || + (message.connectionId && message.connectionId !== edge.connectionId) ) { - const existingIntent = this.viewerRouteIntentsByRoom - .get(participant.roomId) - ?.get(participant.peerId); - if ( - existingIntent?.qualityGuard?.parentPeerId === - assignment.upstream.peerId - ) { - const pending = this.pendingRoutePreparations.get(participant.roomId); - if ( - pending?.intentPeerId === participant.peerId && - pending.routeIntent === existingIntent && - pending.qualityIntent - ) { - pending.qualityIntent.takenOverByRouteFailure = true; - } - existingIntent.failedParentSessionId = this.connectedPeer( - participant.roomId, assignment.upstream.peerId, - )?.sessionId; - existingIntent.failedConnectionId = message.connectionId; - existingIntent.qualityGuard.exclusionOwnedByQuality = false; - existingIntent.qualityGuard = undefined; - } return; } - failedParents.add(assignment.upstream.peerId); - let roomIntents = this.viewerRouteIntentsByRoom.get(participant.roomId); - if (!roomIntents) { - roomIntents = new Map(); - this.viewerRouteIntentsByRoom.set(participant.roomId, roomIntents); - } - roomIntents.set(participant.peerId, { - failedParentPeerId: assignment.upstream.peerId, - failedParentSessionId: this.connectedPeer( - participant.roomId, - assignment.upstream.peerId, - )?.sessionId, - failedConnectionId: message.connectionId, - sessionId: participant.sessionId, - sfuAttempts: 0, - unavailableReported: false, + controller.invalidateEdge({ + childPeerId: participant.peerId, + childSessionId: participant.sessionId, + routeRevision: snapshot.revision, + connectionId: edge.connectionId, + ...(edge.kind === "peer" ? { parentSessionId: edge.parentSessionId } : {}), }); - this.drainViewerRouteIntents(participant.roomId); + this.requestPump(participant.roomId); } - refreshSfu( - participant: AuthenticatedRouteParticipant, - revision: number, - ): void { - void this.sendFreshSfuConfig( - participant.roomId, - participant.peerId, - participant.sessionId, - revision, - true, - true, - ); + refreshSfu(participant: AuthenticatedRouteParticipant, revision: number): void { + const room = this.rooms.get(participant.roomId); + const snapshot = room?.controller?.snapshot(); + if (!room || !snapshot || snapshot.revision !== revision) return; + const key = `${revision}\u0000${participant.peerId}\u0000${participant.sessionId}`; + if (room.refreshKeys.has(key)) return; + room.refreshKeys.add(key); + void this.sendFreshSfuConfig(participant).catch(() => undefined); } - disconnectParticipant(roomId: string, peerId: string): void { - this.clearSelectedEdgeTurnsForParticipant(roomId, peerId); - this.clearViewerQualityStateForParticipant(roomId, peerId); - this.clearDeferredHealthySfuReselectionForParticipant(roomId, peerId); - if (this.peerRelayTopology.getHostPeerId(roomId) === peerId) { - this.scheduleHostOfflineCheck(roomId, peerId); - } - const pending = this.pendingRoutePreparations.get(roomId); - if (!pending || this.pendingSessionsAreCurrent(roomId, pending)) { - return; - } - this.abortPendingRouteForTopologyChange(roomId); + disconnectParticipant(roomId: string, peerId: string, sessionId: string): void { + const room = this.rooms.get(roomId); + if (!room?.controller?.disconnectSession(peerId, sessionId)) return; + if (peerId === room.hostPeerId) this.scheduleHostOfflineCheck(roomId, peerId); + this.requestPump(roomId); } removeViewer(roomId: string, peerId: string): void { - this.clearSelectedEdgeTurnsForParticipant(roomId, peerId); - this.clearViewerQualityStateForParticipant(roomId, peerId); - this.clearDeferredHealthySfuReselectionForParticipant(roomId, peerId); - this.abortPendingRouteForTopologyChange(roomId); - const changes = this.peerRelayTopology.removeViewer( - roomId, - peerId, - this.connectedPeerIds(roomId), - ); - this.clearChangedConnectionIds(roomId, changes); - this.reconcileMediaRoute(roomId); - this.sendMediaAssignmentChanges(roomId, changes); - this.failedParentPeerIdsByViewer.delete(viewerConnectionKey(roomId, peerId)); - this.viewerRouteIntentsByRoom.get(roomId)?.delete(peerId); - this.relayCapacitySessionsByRoom.get(roomId)?.delete(peerId); - this.clearSfuRefreshesForPeer(roomId, peerId); - this.drainViewerRouteIntents(roomId); + const room = this.rooms.get(roomId); + room?.advertisedCapacityByViewer.delete(peerId); + this.options.deleteConnectionId(roomId, peerId); + if (room?.controller?.confirmDeparture(peerId)) this.requestPump(roomId); } stopRoom(roomId: string): void { - this.clearRoomMediaRouteState(roomId); + this.clearRoom(roomId); } deleteRoom(roomId: string): void { - this.peerRelayTopology.deleteRoom(roomId); - this.relayCapacitySessionsByRoom.delete(roomId); - this.clearRoomMediaRouteState(roomId); + this.clearRoom(roomId); } - private handlePrepareRouteFailed( - participant: AuthenticatedRouteParticipant, - message: Extract, + private room(roomId: string): RoomRuntime { + let room = this.rooms.get(roomId); + if (!room) { + room = { + advertisedCapacityByViewer: new Map(), + requested: false, + refreshKeys: new Set(), + }; + this.rooms.set(roomId, room); + } + return room; + } + + private createController( + roomId: string, + room: RoomRuntime, + host: AuthenticatedRouteParticipant, ): void { - const controller = this.mediaRouteControllers.get(participant.roomId); - const pending = this.pendingRoutePreparations.get(participant.roomId); - const pendingRoute = controller?.getPendingRoute(); - if ( - !controller || - !pending || - !pendingRoute || - pending.revision !== message.revision || - pendingRoute.route.revision !== message.revision || - !pending.grantsIssued - ) { - return; - } - const expectedSessionId = pending.expectedSessionIds.get(participant.peerId); - if (!expectedSessionId) { - this.sendError( - participant.sessionId, - "FORBIDDEN", - "Participant is not part of this route plan", - ); - return; - } - if (participant.sessionId !== expectedSessionId) { - return; - } - const publicationGeneration = pendingRoute.route.sfu.publicationGeneration; - const hostOwnsPendingPublication = - participant.role === "host" && - participant.peerId === - this.peerRelayTopology.getHostPeerId(participant.roomId) && - publicationGeneration !== null && - pendingRoute.route.assignments.get(participant.peerId) - ?.sfuPublicationGeneration === publicationGeneration; - if (message.connectionId !== null) { - if ( - hostOwnsPendingPublication && - this.finishSelectedEdgeTurnFailure(participant, message) - ) { - return; - } - this.sendError( - participant.sessionId, - "FORBIDDEN", - "Prepare failure connection is not the selected SFU ingress", - ); - return; - } - if ( - hostOwnsPendingPublication && - (pending.hostSfuIngressTurnAttempted || - this.startSelectedSfuIngressTurn( - participant.roomId, - participant.peerId, - participant.sessionId, - pendingRoute.route, - pending, - )) - ) { - return; - } - this.abortPendingRoute(participant.roomId); - } - - private roomPeerMigrationAttempt( - roomId: string, - viewerPeerId?: string, - ): "healthy" | "quality" | "selected" | null { - const pending = this.pendingRoutePreparations.get(roomId); - if (pending?.healthyReselection) { - return "healthy"; - } - if (pending?.peerQualityReselection) { - return "quality"; - } - const lease = viewerPeerId - ? this.selectedPeerEdgeLease(roomId, viewerPeerId) - : undefined; - return lease ? "selected" : null; - } - - private abortPeerQualityReselectionForHardFailure(roomId: string, failedViewerPeerId?: string): void { - const pending = this.pendingRoutePreparations.get(roomId); - if (pending?.peerQualityReselection && pending.intentPeerId !== failedViewerPeerId) { - this.abortPendingRoute(roomId, undefined, false); - } - } - - private startSelectedEdgeTurn( - roomId: string, - viewerPeerId: string, - requireConsumedSfuRefresh: boolean, - ): boolean { - const config = this.options.selectedEdgeTurn?.config; - const admission = this.options.selectedEdgeTurn?.admission; - const controller = this.mediaRouteControllers.get(roomId); - const active = controller?.getActiveRoute(); - const intent = this.viewerRouteIntentsByRoom.get(roomId)?.get(viewerPeerId); - const roomAttempt = this.roomPeerMigrationAttempt(roomId, viewerPeerId); - if (roomAttempt === "healthy" || roomAttempt === "quality") { - // The room probe already owns recovery, so suppress concurrent failback. - return true; - } - const roomLease = this.selectedPeerEdgeLease(roomId, viewerPeerId); - if (roomLease) { - return true; - } - if ( - !config || - !admission || - !active || - !intent || - intent.selectedTurnAttempted || - !intent.failedParentPeerId || - !intent.failedParentSessionId || - !intent.failedConnectionId - ) { - return false; - } - const viewer = this.options.roomStore.getConnectedViewer( - roomId, - viewerPeerId, - ); - const parent = this.connectedPeer(roomId, intent.failedParentPeerId); - const shareGeneration = this.options.getShareGeneration(roomId); - const parentAssignment = active.assignments.get(intent.failedParentPeerId); - const hostPeerId = this.peerRelayTopology.getHostPeerId(roomId); - const parentSupportsOutboundRelay = - intent.failedParentPeerId === hostPeerId - ? parentAssignment?.upstream.kind === "none" - : parentAssignment?.upstream.kind === "peer"; - const parentChildLimit = - this.peerRelayTopology.getAdvertisedDownstreamCapacity( - roomId, - intent.failedParentPeerId, - ); - const parentActiveEdges = parentAssignment - ? this.activeParentMediaEdges( - roomId, - intent.failedParentPeerId, - parentAssignment, - [viewerPeerId], - ) - : Number.POSITIVE_INFINITY; - if ( - viewer?.sessionId !== intent.sessionId || - parent?.sessionId !== intent.failedParentSessionId || - !parentAssignment || - !parentSupportsOutboundRelay || - parentChildLimit === 0 || - !endpointMediaCopyCountFits(parentActiveEdges, parentChildLimit) || - !shareGeneration - ) { - return false; - } - if (requireConsumedSfuRefresh) { - const publicationGeneration = active.sfu.publicationGeneration; - if ( - !publicationGeneration || - !active.sfu.rootPeerIds.includes(viewerPeerId) || - !this.consumedSfuRefreshesByRoom - .get(roomId) - ?.has( - sfuRefreshKey( - publicationGeneration, - viewerPeerId, - viewer.sessionId, - ), - ) - ) { - return false; - } - } - - intent.selectedTurnAttempted = true; - const newConnectionId = randomBytes(16).toString("base64url"); - const allocationFence: PeerSelectedEdgeTurnIdentity = { - edgeKind: "peer-selected", - roomId, - shareGeneration, - revision: active.revision, - parentPeerId: parent.peerId, - parentSessionId: parent.sessionId, - viewerPeerId, - viewerSessionId: viewer.sessionId, - oldConnectionId: intent.failedConnectionId, - newConnectionId, - }; - if (!admission.reserve(allocationFence)) { - return false; - } - let credential; - try { - credential = issueSelectedEdgeTurnCredential( - config, - allocationFence, - this.options.now?.() ?? Date.now(), - ); - } catch { - this.releaseTurnAllocation(allocationFence); - console.error("Selected-edge TURN credential issuance failed"); - return false; - } - - const timer = setTimeout(() => { - const current = this.selectedPeerEdgeLease(roomId, viewerPeerId); - if ( - !current || - current.newConnectionId !== newConnectionId || - current.answered - ) { - return; - } - const settled = this.settleSelectedPeerEdgeLease(roomId, viewerPeerId); - if (!settled) { - return; - } - this.sendError( - settled.viewerSessionId, - "PEER_NOT_FOUND", - "Selected relay edge timed out", - ); - this.drainViewerRouteIntents(roomId); - }, SELECTED_EDGE_TURN_ATTEMPT_TIMEOUT_MS); - timer.unref(); - const message: Extract = { - type: "selected-edge-turn", - edgeKind: "peer-selected", - revision: active.revision, - parentPeerId: parent.peerId, - viewerPeerId, - oldConnectionId: intent.failedConnectionId, - newConnectionId, - expiresAt: credential.expiresAt, - iceServer: credential.iceServer, - }; - const lease: SelectedPeerEdgeLease = { - roomId, - currentRouteRevision: active.revision, - shareGeneration, - parentPeerId: parent.peerId, - parentSessionId: parent.sessionId, - viewerPeerId, - viewerSessionId: viewer.sessionId, - newConnectionId, - allocationFence, - grant: message, - answered: false, - timer, - }; - this.storeSelectedPeerEdgeLease(lease); - // Queue the Viewer grant first. Any parent offer is handled on a later - // event-loop turn, so the Viewer's socket observes this grant first. - this.options.sendToSession(viewer.sessionId, message); - this.options.sendToSession(parent.sessionId, message); - return true; - } - - /** - * Source-side fallback for a configured hybrid room. This is a LiveKit - * publisher ICE override, not an ordinary peer candidate or a - * participant-wide TURN grant. - */ - private startSelectedSfuIngressTurn( - roomId: string, - hostPeerId: string, - hostSessionId: string, - route: RoomMediaRoute, - pending?: PendingRoutePreparation, - ): boolean { - const config = this.options.selectedEdgeTurn?.config; - const admission = this.options.selectedEdgeTurn?.admission; - const fallback = this.options.sfuFallback; - const key = roomId; - const currentAttempt = this.selectedSfuIngressAttempts.get(key); - if (currentAttempt) { - return ( - currentAttempt.revision === route.revision && - currentAttempt.hostPeerId === hostPeerId && - currentAttempt.hostSessionId === hostSessionId && - currentAttempt.publicationGeneration === - route.sfu.publicationGeneration && - currentAttempt.shareGeneration === - this.options.getShareGeneration(roomId) - ); - } - if ( - !config || - !admission || - !fallback || - (pending !== undefined && - (pending.revision !== route.revision || - pending.hostSfuIngressTurnAttempted || - pending.expectedSessionIds.get(hostPeerId) !== hostSessionId)) - ) { - return false; - } - const revision = route.revision; - const publicationGeneration = route.sfu.publicationGeneration; - const assignment = route.assignments.get(hostPeerId); - const host = this.connectedPeer(roomId, hostPeerId); - const shareGeneration = this.options.getShareGeneration(roomId); - if ( - !publicationGeneration || - assignment?.sfuPublicationGeneration !== publicationGeneration || - !host || - host.sessionId !== hostSessionId || - !shareGeneration - ) { - return false; - } - const activeIngress = this.activeSelectedSfuIngresses.get(roomId); - if ( - activeIngress?.hostPeerId === hostPeerId && - activeIngress.hostSessionId === hostSessionId && - activeIngress.shareGeneration === shareGeneration && - activeIngress.publicationGeneration === publicationGeneration - ) { - return true; - } - - const oldConnectionId = publicationGeneration; - const newConnectionId = randomBytes(16).toString("base64url"); - const allocationFence: HostSfuIngressTurnIdentity = { - edgeKind: "host-sfu-ingress", - roomId, - shareGeneration, - revision, - hostPeerId, - hostSessionId, - publicationGeneration, - oldConnectionId, - newConnectionId, - }; - if (!admission.reserve(allocationFence)) { - return false; - } - let credential; - try { - credential = issueSelectedEdgeTurnCredential( - config, - allocationFence, - this.options.now?.() ?? Date.now(), - ); - } catch { - this.releaseTurnAllocation(allocationFence); - console.error("Selected-edge TURN credential issuance failed"); - return false; - } - - const pendingAttempt = pending; - const timer = setTimeout(() => { - const current = this.selectedSfuIngressAttempts.get(key); - const currentPending = - pendingAttempt !== undefined && - this.pendingRoutePreparations.get(key) === pendingAttempt; - const currentIngress = - current?.newConnectionId === newConnectionId && !current.answered; - if (!currentPending && !currentIngress) { - return; - } - if (currentIngress) { - this.settleSelectedSfuIngressAttempt(key); - this.sendError( - current.hostSessionId, - "PEER_NOT_FOUND", - "Selected SFU relay ingress timed out", - ); - } - if (currentPending) { - this.abortPendingRoute(key); - } else if ( - currentIngress && - this.selectedSfuIngressAttemptIsCurrent(current) - ) { - this.failBackToPeerBaseline(key); - } - }, SELECTED_EDGE_TURN_ATTEMPT_TIMEOUT_MS); - timer.unref(); - const attempt: SelectedSfuIngressAttempt = { - roomId, - revision, - shareGeneration, - hostPeerId, - hostSessionId, - publicationGeneration, - oldConnectionId, - newConnectionId, - allocationFence, - answered: false, - timer, - }; - if (pending) { - clearTimeout(pending.timer); - pending.timer = timer; - pending.hostSfuIngressTurnAttempted = true; - } - this.selectedSfuIngressAttempts.set(key, attempt); - this.options.sendToSession(hostSessionId, { - type: "selected-edge-turn", - edgeKind: "host-sfu-ingress", - revision, - hostPeerId, - publicationGeneration, - oldConnectionId, - newConnectionId, - expiresAt: credential.expiresAt, - iceServer: credential.iceServer, + room.controller = new RoomRouteController({ + hostPeerId: host.peerId, + endpointMediaCopyCapacity: this.options.endpointMediaCopyCapacity, + operationTimeoutMs: + this.options.sfuFallback?.prepareTimeoutMs ?? DEFAULT_ROUTE_OPERATION_TIMEOUT_MS, + selectedTurnEnabled: Boolean(this.options.selectedEdgeTurn), + hostSfuSelectedTurnEnabled: Boolean(this.options.selectedEdgeTurn), + sfuEnabled: Boolean(this.options.sfuFallback), }); - return true; - } - - private markSelectedSfuIngressAnswered( - roomId: string, - hostPeerId: string, - hostSessionId: string, - revision: number, - ): void { - const attempt = this.selectedSfuIngressAttempts.get(roomId); - if ( - !attempt || - attempt.hostPeerId !== hostPeerId || - attempt.hostSessionId !== hostSessionId || - attempt.revision !== revision - ) { - return; - } - if ( - !this.options.selectedEdgeTurn?.admission.commit( - attempt.allocationFence, - ) - ) { - this.settleSelectedSfuIngressAttempt(roomId); - this.sendError( - hostSessionId, - "SERVER_ERROR", - "Selected SFU relay ingress lost its resource reservation", - ); - if (this.pendingRoutePreparations.get(roomId)?.revision === revision) { - this.abortPendingRoute(roomId); - } else { - this.failBackToPeerBaseline(roomId); - } - return; - } - attempt.answered = true; - if (this.pendingRoutePreparations.get(roomId)?.revision === revision) { - return; - } - this.promoteSelectedSfuIngressAttempt(roomId, revision); - } - - private selectedSfuIngressAttemptIsCurrent( - attempt: SelectedSfuIngressAttempt, - ): boolean { - const active = this.mediaRouteControllers - .get(attempt.roomId) - ?.getActiveRoute(); - return ( - active?.revision === attempt.revision && - active.sfu.publicationGeneration === attempt.publicationGeneration && - active.assignments.get(attempt.hostPeerId)?.sfuPublicationGeneration === - attempt.publicationGeneration && - this.connectedPeer(attempt.roomId, attempt.hostPeerId)?.sessionId === - attempt.hostSessionId && - this.options.getShareGeneration(attempt.roomId) === attempt.shareGeneration - ); - } - - private finishSelectedEdgeTurnFailure( - participant: AuthenticatedRouteParticipant, - message: Extract, - ): boolean { - if (participant.role === "host" && message.connectionId !== null) { - const attempt = this.selectedSfuIngressAttempts.get(participant.roomId); - if ( - attempt?.hostPeerId === participant.peerId && - attempt.hostSessionId === participant.sessionId && - attempt.revision === message.revision && - attempt.shareGeneration === - this.options.getShareGeneration(participant.roomId) && - attempt.newConnectionId === message.connectionId - ) { - const pendingRevision = this.pendingRoutePreparations.get( - participant.roomId, - )?.revision; - this.settleSelectedSfuIngressAttempt(participant.roomId); - this.sendError( - participant.sessionId, - "PEER_NOT_FOUND", - "Selected SFU relay ingress failed", - ); - if (pendingRevision === message.revision) { - this.abortPendingRoute(participant.roomId, undefined, false); - } else { - this.failBackToPeerBaseline(participant.roomId); - } - return true; - } - const activeIngress = this.activeSelectedSfuIngresses.get( - participant.roomId, - ); - const activeRoute = this.mediaRouteControllers - .get(participant.roomId) - ?.getActiveRoute(); - if ( - activeIngress?.hostPeerId === participant.peerId && - activeIngress.hostSessionId === participant.sessionId && - activeIngress.connectionId === message.connectionId && - activeIngress.shareGeneration === - this.options.getShareGeneration(participant.roomId) && - activeRoute !== undefined && - activeRoute.sfu.publicationGeneration === - activeIngress.publicationGeneration && - activeRoute.assignments.get(participant.peerId) - ?.sfuPublicationGeneration === activeIngress.publicationGeneration && - ((message.phase === "active" && - activeRoute.revision === message.revision) || - (message.phase === "prepare" && - this.pendingRoutePreparations.get(participant.roomId)?.revision === - message.revision)) - ) { - this.settleActiveSelectedSfuIngress(participant.roomId); - this.sendError( - participant.sessionId, - "PEER_NOT_FOUND", - "Selected SFU relay ingress failed", - ); - this.failBackToPeerBaseline(participant.roomId); - return true; - } - } - if (message.connectionId === null) { - return false; - } - const lease = this.selectedPeerEdgeLeaseForParticipantConnection( - participant.roomId, - participant.peerId, - message.connectionId, - ); - const active = this.mediaRouteControllers - .get(participant.roomId) - ?.getActiveRoute(); - const participantOwnsLease = - (lease?.viewerPeerId === participant.peerId && - lease.viewerSessionId === participant.sessionId) || - (lease?.parentPeerId === participant.peerId && - lease.parentSessionId === participant.sessionId); - if ( - !lease || - !active || - !participantOwnsLease || - lease.newConnectionId !== message.connectionId || - !this.selectedPeerEdgeLeaseIsCurrent( - participant.roomId, - lease, - active, - ) - ) { - return false; - } - this.abortPeerQualityReselectionForHardFailure( - participant.roomId, - lease.viewerPeerId, - ); - this.settleSelectedPeerEdgeLease( - participant.roomId, - lease.viewerPeerId, - ); - this.sendError( - participant.sessionId, - "PEER_NOT_FOUND", - "Selected relay edge failed", - ); - this.drainViewerRouteIntents(participant.roomId); - return true; - } - - private failBackToPeerBaseline(roomId: string): void { - this.abortPendingRoute(roomId, undefined, false); - const controller = this.mediaRouteControllers.get(roomId); - const hostPeerId = this.peerRelayTopology.getHostPeerId(roomId); - if (!controller || !hostPeerId) { - return; - } - const before = controller.getActiveRoute(); - const fillChanges = this.peerRelayTopology.setHost( - roomId, - hostPeerId, - this.connectedPeerIds(roomId), - ); - this.clearChangedConnectionIds(roomId, fillChanges); - const assignments = participantRoutesFromTopology( - this.peerRelayTopology.getAssignments(roomId), - hostPeerId, - null, - [], - this.endpointMediaCopyCapacity, - ); - const revision = controller.reconcileBaseline({ - assignments, - sfuPublicationGeneration: null, - sfuRootPeerIds: [], + room.controller.upsertParticipant({ + peerId: host.peerId, + role: "host", + sessionId: host.sessionId, + effectiveDownstreamCapacity: this.options.endpointMediaCopyCapacity, }); - if (revision === undefined) { - return; - } - this.sfuDisabledRoomIds.add(roomId); - this.clearRoomFailedParents(roomId); - const active = controller.getActiveRoute(); - this.consumedSfuRefreshesByRoom.delete(roomId); - this.clearChangedRouteConnectionIds(roomId, before, active); - this.broadcastActiveRoute(roomId, active); - this.drainViewerRouteIntents(roomId); - } - - private retireShrinkingSfuRoute( - roomId: string, - survivingRootPeerIds: readonly string[], - ): void { - this.abortPendingRoute(roomId, undefined, false); - const controller = this.mediaRouteControllers.get(roomId); - const hostPeerId = this.peerRelayTopology.getHostPeerId(roomId); - if (!controller || !hostPeerId) { - return; - } - const before = controller.getActiveRoute(); - const assignments = participantRoutesFromTopology( - this.peerRelayTopology.getAssignments(roomId), - hostPeerId, - null, - [], - this.endpointMediaCopyCapacity, - ); - const revision = controller.reconcileBaseline({ - assignments, - sfuPublicationGeneration: null, - sfuRootPeerIds: [], - }); - if (revision === undefined) { - return; - } - this.consumedSfuRefreshesByRoom.delete(roomId); - const active = controller.getActiveRoute(); - this.clearChangedRouteConnectionIds(roomId, before, active); - this.broadcastActiveRoute(roomId, active); - - for (const peerId of survivingRootPeerIds) { - const viewer = this.options.roomStore.getConnectedViewer(roomId, peerId); - const assignment = active.assignments.get(peerId); - if (!viewer || !assignment || assignment.upstream.kind === "sfu") { - continue; - } - const failedParentPeerId = - assignment.upstream.kind === "peer" && - this.failedParentPeerIdsByViewer - .get(viewerConnectionKey(roomId, peerId)) - ?.has(assignment.upstream.peerId) - ? assignment.upstream.peerId - : assignment.upstream.kind === "none" - ? null - : undefined; - if (failedParentPeerId === undefined) { - continue; - } - let intents = this.viewerRouteIntentsByRoom.get(roomId); - if (!intents) { - intents = new Map(); - this.viewerRouteIntentsByRoom.set(roomId, intents); - } - intents.set(peerId, { - failedParentPeerId, + for (const viewer of this.options.roomStore.getConnectedViewers(roomId)) { + room.controller.upsertParticipant({ + peerId: viewer.peerId, + role: "viewer", sessionId: viewer.sessionId, - sfuAttempts: 0, - unavailableReported: false, + effectiveDownstreamCapacity: + room.advertisedCapacityByViewer.get(viewer.peerId) ?? 0, }); } - this.drainViewerRouteIntents(roomId); } - private drainViewerRouteIntents(roomId: string): void { - this.enqueueUnassignedViewerIntents(roomId); - const pending = this.pendingRoutePreparations.get(roomId); - if (pending) { - if (this.abortStalePendingQualityIntent(roomId, pending)) { + private requestPump(roomId: string): void { + const room = this.rooms.get(roomId); + if (!room?.controller || this.closing) return; + room.requested = true; + if (room.pump) return; + room.pump = (async () => { + try { + while (room.requested && !this.closing) { + room.requested = false; + await this.pumpRoom(roomId, room); + } + } finally { + room.pump = undefined; + if (room.requested && !this.closing) this.requestPump(roomId); + } + })().catch(() => { + console.error("Media route reconciliation failed"); + this.sendRoomError(roomId, "SERVER_ERROR", "Media routing failed"); + }); + } + + private async pumpRoom(roomId: string, room: RoomRuntime): Promise { + const controller = room.controller; + if (!controller) return; + for (;;) { + const beforeRevision = controller.snapshot().revision; + const reconciled = controller.reconcile(this.now()); + this.releaseResources(reconciled.released); + if (controller.snapshot().revision !== beforeRevision) this.broadcastActive(roomId, room); + const operation = reconciled.operation ?? controller.snapshot().operation; + if (!operation) { + this.clearDeadline(room); return; } + if (operation.current) { + this.scheduleDeadline(roomId, room, operation); + return; + } + const plan = operation.candidates[operation.cursor]; + if (!plan) return; + const guard = cursorGuard(operation, plan); + const preparation = await this.prepareCandidate( + roomId, + room, + operation, + plan, + operation.baseRevision + + (plan.endpointTransition.kind === "bounded-gap" ? 2 : 1), + ); + if (preparation.kind === "denied") { + if (preparation.serverAdmission === "sfu") { + this.resourceWaiterRoomIds.add(roomId); + this.scheduleDeadline(roomId, room, operation); + return; + } + const skipped = controller.skipCurrentCandidate(guard, this.now()); + this.releaseResources(skipped.released); + if (skipped.exhausted) { + this.sendViewerError( + roomId, + operation.childPeerId, + "PEER_NOT_FOUND", + "No usable media route is available", + ); + } + if (preparation.serverAdmission === "turn" && skipped.exhausted) { + this.resourceWaiterRoomIds.add(roomId); + } + continue; + } + let currentOperation = controller.snapshot().operation; + if (plan.endpointTransition.kind === "bounded-gap") { + const beforeGap = controller.snapshot().revision; + const gap = controller.retireCurrentCandidateProducer(guard, this.now()); + this.releaseResources(gap.released); + if (!gap.accepted) { + this.releaseReservation(preparation.prepared.reservation); + continue; + } + if (controller.snapshot().revision !== beforeGap) this.broadcastActive(roomId, room); + currentOperation = controller.snapshot().operation; + } + const currentPlan = currentOperation?.candidates[currentOperation.cursor]; + if (!currentOperation || !currentPlan) { + this.releaseReservation(preparation.prepared.reservation); + continue; + } + const begun = controller.beginCurrentCandidate({ + guard: cursorGuard(currentOperation, currentPlan), + nowMs: this.now(), + connectionId: preparation.prepared.connectionId, + reservation: preparation.prepared.reservation, + publicationGeneration: preparation.prepared.publicationGeneration, + publicationConnectionId: preparation.prepared.publicationConnectionId, + }); + this.releaseResources(begun.released); + if (!begun.accepted || !begun.operation?.current) { + if (begun.exhausted) { + this.sendViewerError( + roomId, + operation.childPeerId, + "PEER_NOT_FOUND", + "No usable media route is available", + ); + } + continue; + } + this.sendPrepareMessages( + roomId, + room, + begun.operation, + preparation.prepared, + ); + this.scheduleDeadline(roomId, room, begun.operation); return; } - if (this.pendingRoutePreparations.has(roomId)) { - return; + } + + private async prepareCandidate( + roomId: string, + room: RoomRuntime, + operation: OperationSnapshot, + plan: CandidatePlan, + revision: number, + ): Promise { + const snapshot = room.controller!.snapshot(); + const child = this.options.roomStore.getConnectedViewer(roomId, operation.childPeerId); + const shareGeneration = this.options.getShareGeneration(roomId); + if (!child || child.sessionId !== operation.childSessionId || !shareGeneration) { + return { kind: "denied", serverAdmission: false }; } - const intents = this.viewerRouteIntentsByRoom.get(roomId); - if (!intents) { - return; + const connectionId = opaqueId(); + const overlap = + plan.endpointTransition.kind === "overlap" + ? overlapResource(plan.endpointTransition.producerPeerId) + : undefined; + if (plan.tuple.kind === "peer") { + if (plan.tuple.transport === "direct") { + return { + kind: "ready", + prepared: { + connectionId, + reservation: { kind: "direct", ...(overlap ? { overlap } : {}) }, + }, + }; + } + const parent = this.connectedPeer(roomId, plan.tuple.parentPeerId); + const selected = this.options.selectedEdgeTurn; + if (!parent || !selected) return { kind: "denied", serverAdmission: false }; + const oldConnectionId = + snapshot.upstreamByViewer.get(operation.childPeerId)?.connectionId ?? connectionId; + const fence: PeerSelectedEdgeTurnIdentity = { + edgeKind: "peer-selected", + roomId, + shareGeneration, + revision, + parentPeerId: parent.peerId, + parentSessionId: parent.sessionId, + viewerPeerId: operation.childPeerId, + viewerSessionId: operation.childSessionId, + oldConnectionId, + newConnectionId: connectionId, + }; + if (!selected.admission.reserve(fence)) { + return { kind: "denied", serverAdmission: "turn" }; + } + const resource = turnResource(fence); + try { + const credential = issueSelectedEdgeTurnCredential(selected.config, fence, this.now()); + return { + kind: "ready", + prepared: { + connectionId, + reservation: { + kind: "selected-turn", + edge: resource, + ...(overlap ? { overlap } : {}), + }, + peerGrant: { + type: "selected-edge-turn", + edgeKind: "peer-selected", + revision, + parentPeerId: parent.peerId, + viewerPeerId: operation.childPeerId, + oldConnectionId, + newConnectionId: connectionId, + expiresAt: credential.expiresAt, + iceServer: credential.iceServer, + }, + }, + }; + } catch { + this.releaseResource(resource); + return { kind: "denied", serverAdmission: false }; + } } - for (const [viewerPeerId, intent] of [...intents]) { + const fallback = this.options.sfuFallback; + const host = this.options.roomStore.getConnectedHost(roomId); + if (!fallback || !host || host.peerId !== room.hostPeerId) { + return { kind: "denied", serverAdmission: false }; + } + if (plan.tuple.publication === "reuse") { + const publication = snapshot.hostPublication; if ( - this.selectedPeerEdgeLease(roomId, viewerPeerId) + !publication || + publication.resource.kind !== "sfu-publication" ) { - continue; + return { kind: "denied", serverAdmission: false }; } - if ( - intent.qualityGuard && - !this.qualityIntentIsCurrent(roomId, viewerPeerId, intent) - ) { - this.discardQualityIntent(roomId, viewerPeerId, intent); - continue; + const fence: SfuSubscriptionFence = { + ...publication.resource.fence, + viewerPeerId: operation.childPeerId, + }; + if (!fallback.admission.reserveSubscription(fence)) { + return { kind: "denied", serverAdmission: "sfu" }; } - const viewer = this.options.roomStore.getConnectedViewer( - roomId, - viewerPeerId, - ); - this.pruneHeldQualityParentExclusion( - roomId, - viewerPeerId, - this.options.now?.() ?? Date.now(), - ); - const controller = this.mediaRouteControllers.get(roomId); - const assignment = controller - ?.getActiveRoute() - .assignments.get(viewerPeerId); - if (viewer?.sessionId !== intent.sessionId || !assignment) { - if (intent.qualityGuard) { - this.discardQualityIntent(roomId, viewerPeerId, intent); - } else { - intents.delete(viewerPeerId); - } - continue; - } - if (intent.failedParentPeerId === null) { - if (assignment.upstream.kind !== "none") { - intents.delete(viewerPeerId); - continue; - } - const active = controller!.getActiveRoute(); - const excludedParents = new Set(); - const baselineParentPeerId = this.peerRelayTopology.getAssignment( + const currentEdge = snapshot.upstreamByViewer.get(operation.childPeerId); + const edge = + currentEdge?.kind === "sfu" && + currentEdge.publicationGeneration === publication.generation && + currentEdge.resource.kind === "sfu-subscription" && + !currentEdge.resource.released + ? currentEdge.resource + : subscriptionResource(fence); + try { + const token = await fallback.tokenIssuer.issueToken({ roomId, - viewerPeerId, - )?.parentPeerId; - if (baselineParentPeerId) { - excludedParents.add(baselineParentPeerId); - } - const hostPeerId = this.peerRelayTopology.getHostPeerId(roomId); - if ( - active.sfu.publicationGeneration && - hostPeerId && - (active.assignments.get(hostPeerId)?.childPeerIds.length ?? 0) >= 1 - ) { - excludedParents.add(hostPeerId); - } - if ( - intent.qualityGuard && - !this.qualityIntentIsCurrent(roomId, viewerPeerId, intent) - ) { - this.discardQualityIntent(roomId, viewerPeerId, intent); - continue; - } - const changes = this.peerRelayTopology.reassignViewer( + role: "viewer", + peerId: operation.childPeerId, + shareGeneration, + publicationGeneration: publication.generation, + allowlistedRootPeerIds: [operation.childPeerId], + }); + return { + kind: "ready", + prepared: { + connectionId, + reservation: { + kind: "sfu-reuse", + edge, + ...(overlap ? { overlap } : {}), + }, + publicationGeneration: publication.generation, + viewerSfuConfig: { + type: "sfu-config", + revision, + url: fallback.url, + token, + }, + }, + }; + } catch { + fallback.admission.releaseSubscription(fence); + return { kind: "denied", serverAdmission: false }; + } + } + + return await this.prepareNewPublication( + roomId, + room, + operation, + plan, + revision, + connectionId, + shareGeneration, + host, + overlap, + ); + } + + private async prepareNewPublication( + roomId: string, + room: RoomRuntime, + operation: OperationSnapshot, + plan: CandidatePlan, + revision: number, + connectionId: string, + shareGeneration: string, + host: { peerId: string; sessionId: string }, + overlap?: OverlapRouteResource, + ): Promise { + const fallback = this.options.sfuFallback!; + const publicationGeneration = opaqueId(); + const publicationFence: SfuResourceFence = { + roomId, + shareGeneration, + publicationGeneration, + }; + const subscriptionFence: SfuSubscriptionFence = { + ...publicationFence, + viewerPeerId: operation.childPeerId, + }; + if (!fallback.admission.reservePublication(publicationFence)) { + return { kind: "denied", serverAdmission: "sfu" }; + } + if (!fallback.admission.reserveSubscription(subscriptionFence)) { + fallback.admission.beginDrain(publicationFence); + this.scheduleSfuDrain(publicationFence); + return { kind: "denied", serverAdmission: "sfu" }; + } + const subscription = subscriptionResource(subscriptionFence); + let ingressTurn: TurnRouteResource | undefined; + let hostIngressGrant: PreparedCandidate["hostIngressGrant"]; + const publicationConnectionId = + plan.tuple.kind === "sfu" && plan.tuple.ingress === "selected-turn" + ? connectionId + : publicationGeneration; + if (plan.tuple.kind === "sfu" && plan.tuple.ingress === "selected-turn") { + const selected = this.options.selectedEdgeTurn; + if (!selected) { + this.releaseResource(subscription); + fallback.admission.beginDrain(publicationFence); + this.scheduleSfuDrain(publicationFence); + return { kind: "denied", serverAdmission: false }; + } + const fence: HostSfuIngressTurnIdentity = { + edgeKind: "host-sfu-ingress", + roomId, + shareGeneration, + revision, + hostPeerId: host.peerId, + hostSessionId: host.sessionId, + publicationGeneration, + oldConnectionId: + room.controller?.snapshot().hostPublication?.connectionId ?? publicationGeneration, + newConnectionId: connectionId, + }; + if (!selected.admission.reserve(fence)) { + this.releaseResource(subscription); + fallback.admission.beginDrain(publicationFence); + this.scheduleSfuDrain(publicationFence); + return { kind: "denied", serverAdmission: "turn" }; + } + ingressTurn = turnResource(fence); + try { + const credential = issueSelectedEdgeTurnCredential(selected.config, fence, this.now()); + hostIngressGrant = { + type: "selected-edge-turn", + edgeKind: "host-sfu-ingress", + revision, + hostPeerId: host.peerId, + publicationGeneration, + oldConnectionId: fence.oldConnectionId, + newConnectionId: connectionId, + expiresAt: credential.expiresAt, + iceServer: credential.iceServer, + }; + } catch { + this.releaseResource(ingressTurn); + this.releaseResource(subscription); + fallback.admission.beginDrain(publicationFence); + this.scheduleSfuDrain(publicationFence); + return { kind: "denied", serverAdmission: false }; + } + } + const publication = publicationResource(publicationFence, ingressTurn); + try { + await fallback.roomControl.createRoom(publicationFence); + const [hostToken, viewerToken] = await Promise.all([ + fallback.tokenIssuer.issueToken({ roomId, - viewerPeerId, - this.connectedPeerIds(roomId), - excludedParents, - MAX_PEER_RELAY_DEPTH, - ); - if (changes) { - if (intent.qualityGuard) { - this.startRoomQualityMigrationCooldown(roomId); - this.releaseQualityOwnedParentExclusion( - roomId, - viewerPeerId, - intent, - intent.qualityGuard, - ); - } - intents.delete(viewerPeerId); - this.clearChangedConnectionIds(roomId, changes); - this.reconcileMediaRoute(roomId); - this.sendMediaAssignmentChanges(roomId, changes); - continue; + role: "host", + peerId: host.peerId, + shareGeneration, + publicationGeneration, + allowlistedRootPeerIds: [operation.childPeerId], + }), + fallback.tokenIssuer.issueToken({ + roomId, + role: "viewer", + peerId: operation.childPeerId, + shareGeneration, + publicationGeneration, + allowlistedRootPeerIds: [operation.childPeerId], + }), + ]); + return { + kind: "ready", + prepared: { + connectionId, + publicationGeneration, + publicationConnectionId, + reservation: { + kind: "sfu-create", + edge: subscription, + publication, + ...(overlap ? { overlap } : {}), + }, + hostIngressGrant, + hostSfuConfig: { + type: "sfu-config", + revision, + url: fallback.url, + token: hostToken, + }, + viewerSfuConfig: { + type: "sfu-config", + revision, + url: fallback.url, + token: viewerToken, + }, + }, + }; + } catch { + this.releaseResource(subscription); + this.releaseResource(publication); + return { kind: "denied", serverAdmission: false }; + } + } + + private sendPrepareMessages( + roomId: string, + room: RoomRuntime, + operation: OperationSnapshot, + prepared: PreparedCandidate, + ): void { + const current = operation.current!; + const candidate = preparedRouteCandidate(operation, current.tuple, current.connectionId); + const assignments = this.assignments( + roomId, + room.controller!.snapshot(), + current.tuple, + operation.childPeerId, + prepared.publicationGeneration, + ); + const child = this.options.roomStore.getConnectedViewer(roomId, operation.childPeerId); + if (!child || child.sessionId !== operation.childSessionId) return; + const childUpdate: ServerMessage = { + type: "route-update", + revision: current.revision, + phase: "prepare", + assignment: assignments.get(operation.childPeerId) ?? emptyAssignment(), + candidate, + }; + if (current.tuple.kind === "peer") { + const parent = this.connectedPeer(roomId, current.tuple.parentPeerId); + if (!parent) return; + this.options.sendToSession(child.sessionId, childUpdate); + if (prepared.peerGrant) this.options.sendToSession(child.sessionId, prepared.peerGrant); + this.options.sendToSession(parent.sessionId, { + type: "route-update", + revision: current.revision, + phase: "prepare", + assignment: assignments.get(parent.peerId) ?? emptyAssignment(), + candidate, + }); + if (prepared.peerGrant) this.options.sendToSession(parent.sessionId, prepared.peerGrant); + return; + } + const host = this.options.roomStore.getConnectedHost(roomId); + if (prepared.hostSfuConfig && host) { + this.options.sendToSession(host.sessionId, { + type: "route-update", + revision: current.revision, + phase: "prepare", + assignment: assignments.get(host.peerId) ?? emptyAssignment(), + candidate, + }); + if (prepared.hostIngressGrant) { + this.options.sendToSession(host.sessionId, prepared.hostIngressGrant); + } + this.options.sendToSession(host.sessionId, prepared.hostSfuConfig); + } + this.options.sendToSession(child.sessionId, childUpdate); + if (prepared.viewerSfuConfig) { + this.options.sendToSession(child.sessionId, prepared.viewerSfuConfig); + } + } + + private assignments( + roomId: string, + snapshot: RouteSnapshot, + candidateTuple?: CandidateTuple, + candidateChildPeerId?: string, + candidatePublicationGeneration?: string, + ): Map { + const hostPeerId = this.rooms.get(roomId)?.hostPeerId; + if (!hostPeerId) return new Map(); + const peerIds = new Set([hostPeerId, ...this.options.roomStore.getViewerPeerIds(roomId)]); + const edges = new Map< + string, + { kind: "peer"; parentPeerId: string } | { kind: "sfu"; publicationGeneration: string } + >(); + for (const [childPeerId, edge] of snapshot.upstreamByViewer) { + if (!edge.physicalActive) continue; + edges.set( + childPeerId, + edge.kind === "peer" + ? { kind: "peer", parentPeerId: edge.parentPeerId } + : { kind: "sfu", publicationGeneration: edge.publicationGeneration }, + ); + } + let publicationGeneration = snapshot.hostPublication?.physicalActive + ? snapshot.hostPublication.generation + : null; + if (candidateTuple && candidateChildPeerId) { + edges.delete(candidateChildPeerId); + if (candidateTuple.kind === "peer") { + edges.set(candidateChildPeerId, { + kind: "peer", + parentPeerId: candidateTuple.parentPeerId, + }); + const old = snapshot.upstreamByViewer.get(candidateChildPeerId); + if ( + old?.kind === "sfu" && + [...edges.values()].every((edge) => edge.kind !== "sfu") + ) { + publicationGeneration = null; } } else { - if (assignment.upstream.kind === "sfu") { - continue; - } - if ( - assignment.upstream.kind !== "peer" || - assignment.upstream.peerId !== intent.failedParentPeerId - ) { - intents.delete(viewerPeerId); - continue; - } - const failedParents = this.failedParentPeerIdsByViewer.get( - viewerConnectionKey(roomId, viewerPeerId), - ); - if (!failedParents?.has(intent.failedParentPeerId)) { - if (intent.qualityGuard) { - this.discardQualityIntent(roomId, viewerPeerId, intent); - } else { - intents.delete(viewerPeerId); + if (candidateTuple.publication !== "reuse") { + for (const [peerId, edge] of [...edges]) { + if (edge.kind === "sfu") edges.delete(peerId); } - continue; + publicationGeneration = candidatePublicationGeneration ?? null; } - if ( - intent.qualityGuard && - !this.qualityIntentIsCurrent(roomId, viewerPeerId, intent) - ) { - this.discardQualityIntent(roomId, viewerPeerId, intent); - continue; + if (publicationGeneration) { + edges.set(candidateChildPeerId, { + kind: "sfu", + publicationGeneration, + }); } - if (intent.qualityGuard) { - if ( - this.startPeerQualityReselection( - roomId, - viewerPeerId, - intent, - failedParents, - ) - ) { - return; - } - this.discardQualityIntent(roomId, viewerPeerId, intent, true); - continue; - } - const changes = this.peerRelayTopology.reassignViewer( - roomId, - viewerPeerId, - this.connectedPeerIds(roomId), - failedParents, - MAX_PEER_RELAY_DEPTH, - ); - if (changes) { - intents.delete(viewerPeerId); - this.clearChangedConnectionIds(roomId, changes); - this.reconcileMediaRoute(roomId); - this.sendMediaAssignmentChanges(roomId, changes); - continue; - } - } - - if (intent.qualityGuard?.kind === "relative-fps") { - this.discardQualityIntent(roomId, viewerPeerId, intent); - continue; - } - - if ( - intent.sfuAttempts === 0 && - this.options.sfuFallback && - !this.sfuDisabledRoomIds.has(roomId) - ) { - if ( - intent.qualityGuard && - !this.qualityIntentIsCurrent(roomId, viewerPeerId, intent) - ) { - this.discardQualityIntent(roomId, viewerPeerId, intent); - continue; - } - const result = this.prepareSfuFallback( - roomId, - viewerPeerId, - intent.sessionId, - intent, - ); - if (result === "started") { - if (intent.qualityGuard) { - this.startRoomQualityMigrationCooldown(roomId); - } - intent.sfuAttempts += 1; - return; - } - if (result === "busy") { - return; - } - intent.unavailableReported = true; - } - - if ( - (intent.sfuAttempts > 0 || intent.unavailableReported) && - this.startSelectedEdgeTurn(roomId, viewerPeerId, false) - ) { - if (this.selectedPeerEdgeLease(roomId, viewerPeerId)) { - continue; - } - return; - } - - if (!intent.unavailableReported) { - this.sendError( - intent.sessionId, - "PEER_NOT_FOUND", - "No media fallback route is available", - ); - intent.unavailableReported = true; } } - if (intents.size === 0) { - this.viewerRouteIntentsByRoom.delete(roomId); + const sourceReachable = (peerId: string, seen = new Set()): boolean => { + if (peerId === hostPeerId) return true; + if (seen.has(peerId)) return false; + seen.add(peerId); + const edge = edges.get(peerId); + if (!edge) return false; + return edge.kind === "sfu" + ? publicationGeneration === edge.publicationGeneration + : sourceReachable(edge.parentPeerId, seen); + }; + const assignments = new Map(); + for (const peerId of peerIds) { + const edge = edges.get(peerId); + const reachable = sourceReachable(peerId); + const childPeerIds = [...edges] + .filter(([, candidate]) => + candidate.kind === "peer" && + candidate.parentPeerId === peerId && + sourceReachable(candidate.parentPeerId), + ) + .map(([childPeerId]) => childPeerId); + assignments.set(peerId, { + upstream: + peerId === hostPeerId || !reachable || !edge + ? { kind: "none" } + : edge.kind === "peer" + ? { kind: "peer", peerId: edge.parentPeerId } + : { kind: "sfu" }, + childPeerIds, + sfuPublicationGeneration: + peerId === hostPeerId ? publicationGeneration : null, + }); } + return assignments; } - private startPeerQualityReselection( - roomId: string, - viewerPeerId: string, - intent: ViewerRouteIntent, - excludedParentPeerIds: ReadonlySet, - ): boolean { - const guard = intent.qualityGuard; - const controller = this.mediaRouteControllers.get(roomId); - const active = controller?.getActiveRoute(); - const viewer = this.options.roomStore.getConnectedViewer( - roomId, - viewerPeerId, - ); - const shareGeneration = this.options.getShareGeneration(roomId); - if ( - !guard || - !controller || - controller.getPendingRoute() || - this.roomPeerMigrationAttempt(roomId, viewerPeerId) !== null || - !active || - !viewer || - !shareGeneration || - active.revision !== guard.routeRevision || - !this.qualityIntentIsCurrent(roomId, viewerPeerId, intent) - ) { - return false; - } - - const candidateExclusions = new Set(excludedParentPeerIds); - const hostPeerId = this.peerRelayTopology.getHostPeerId(roomId); - for (const [peerId, assignment] of active.assignments) { - const isHost = peerId === hostPeerId; - const participant = isHost - ? this.options.roomStore.getConnectedHost(roomId) - : this.options.roomStore.getConnectedViewer(roomId, peerId); - const supportedParent = isHost - ? assignment.upstream.kind === "none" - : assignment.upstream.kind === "peer"; - if ( - !participant || - !supportedParent || - !endpointMediaCopyCountFits( - this.activeParentMediaEdges(roomId, peerId, assignment) + 1, - this.peerRelayTopology.getDownstreamCapacity(roomId, peerId), - "transition", - ) - ) { - candidateExclusions.add(peerId); - } - } - const newParentPeerId = this.peerRelayTopology.findViewerReassignmentParent( - roomId, - viewerPeerId, - this.connectedPeerIds(roomId), - candidateExclusions, - MAX_PEER_RELAY_DEPTH, - ); - const newParent = newParentPeerId - ? this.connectedPeer(roomId, newParentPeerId) - : undefined; - if (!newParentPeerId || !newParent) { - return false; - } - - const assignments = cloneAssignments(active.assignments); - const participantOrder = [...assignments.keys()]; - const viewerAssignment = assignments.get(viewerPeerId); - const oldParentAssignment = assignments.get(guard.parentPeerId); - const newParentAssignment = assignments.get(newParentPeerId); - if ( - viewerAssignment?.upstream.kind !== "peer" || - viewerAssignment.upstream.peerId !== guard.parentPeerId || - !oldParentAssignment || - !newParentAssignment - ) { - return false; - } - assignments.set(guard.parentPeerId, { - ...oldParentAssignment, - childPeerIds: oldParentAssignment.childPeerIds.filter( - (peerId) => peerId !== viewerPeerId, - ), - }); - assignments.set(newParentPeerId, { - ...newParentAssignment, - childPeerIds: [...newParentAssignment.childPeerIds, viewerPeerId].sort((left, right) => - participantOrder.indexOf(left) - participantOrder.indexOf(right)), - }); - assignments.set(viewerPeerId, { - ...viewerAssignment, - upstream: { kind: "peer", peerId: newParentPeerId }, - }); - - let revision: number | undefined; - try { - revision = controller.prepare({ - assignments, - expectedParticipantIds: new Set([viewerPeerId]), - sfuPublicationGeneration: active.sfu.publicationGeneration, - sfuRootPeerIds: active.sfu.rootPeerIds, - }); - } catch { - return false; - } - if (revision === undefined) { - return false; - } - const timer = setTimeout( - () => this.abortPendingRoute(roomId), - this.options.sfuFallback?.prepareTimeoutMs ?? DEFAULT_SFU_PREPARE_TIMEOUT_MS, - ); - timer.unref(); - this.pendingRoutePreparations.set(roomId, { - revision, - intentPeerId: viewerPeerId, - routeIntent: intent, - qualityIntent: { - guard: { ...guard }, - takenOverByRouteFailure: false, - }, - expectedSessionIds: new Map([ - [viewerPeerId, viewer.sessionId], - [newParentPeerId, newParent.sessionId], - ]), - grantsIssued: true, - hostSfuIngressTurnAttempted: false, - healthyReselection: null, - peerQualityReselection: { - activeRevision: active.revision, - shareGeneration, - excludedParentPeerIds: candidateExclusions, - newParentPeerId, - newConnectionId: null, - }, - sfuResourceFence: null, - timer, - }); - this.startRoomQualityMigrationCooldown(roomId); - for (const [sessionId, assignment] of [ - [viewer.sessionId, assignments.get(viewerPeerId)!], - [newParent.sessionId, assignments.get(newParentPeerId)!], - ] as const) { - this.options.sendToSession(sessionId, { + private broadcastActive(roomId: string, room: RoomRuntime): void { + const snapshot = room.controller?.snapshot(); + if (!snapshot) return; + const assignments = this.assignments(roomId, snapshot); + const participants = [ + this.options.roomStore.getConnectedHost(roomId), + ...this.options.roomStore.getConnectedViewers(roomId), + ].filter((peer): peer is { peerId: string; sessionId: string } => Boolean(peer)); + for (const participant of participants) { + const assignment = assignments.get(participant.peerId) ?? emptyAssignment(); + this.options.sendToSession(participant.sessionId, { type: "route-update", - revision, - phase: "prepare", + revision: snapshot.revision, + phase: "active", assignment, }); - } - return true; - } - - private qualityIntentIsCurrent( - roomId: string, - viewerPeerId: string, - intent: ViewerRouteIntent, - ): boolean { - const guard = intent.qualityGuard; - if (!guard) { - return true; - } - const edge = this.resolveActivePeerEdge(roomId, viewerPeerId); - return ( - this.options.roomStore.getConnectedViewer(roomId, viewerPeerId) - ?.sessionId === guard.viewerSessionId && - this.connectedPeer(roomId, guard.parentPeerId)?.sessionId === - guard.parentSessionId && - edge?.parentPeerId === guard.parentPeerId && - edge.revision === guard.routeRevision && - this.options.getConnectionId(roomId, viewerPeerId) === guard.connectionId - ); - } - - private discardQualityIntent( - roomId: string, - viewerPeerId: string, - intent: ViewerRouteIntent, - holdParentExclusion = false, - ): void { - const guard = intent.qualityGuard; - const intents = this.viewerRouteIntentsByRoom.get(roomId); - if (!guard || intents?.get(viewerPeerId) !== intent) { - return; - } - if (holdParentExclusion) { - this.heldQualityParentExclusionsByViewer.set( - viewerConnectionKey(roomId, viewerPeerId), - { - parentPeerId: guard.parentPeerId, - parentSessionId: guard.parentSessionId, - expiresAtMs: - (this.options.now?.() ?? Date.now()) + - VIEWER_QUALITY_REASSIGN_COOLDOWN_MS, - }, - ); - guard.exclusionOwnedByQuality = false; - } - intents.delete(viewerPeerId); - if (intents.size === 0) { - this.viewerRouteIntentsByRoom.delete(roomId); - } - this.releaseQualityOwnedParentExclusion( - roomId, - viewerPeerId, - intent, - guard, - ); - this.viewerQualityEvidenceStates.delete( - viewerConnectionKey(roomId, viewerPeerId), - ); - } - - private releaseQualityOwnedParentExclusion( - roomId: string, - viewerPeerId: string, - excludedIntent: ViewerRouteIntent, - guard: QualityRouteIntentGuard, - ): void { - if (!guard.exclusionOwnedByQuality) { - return; - } - const currentIntent = this.viewerRouteIntentsByRoom - .get(roomId) - ?.get(viewerPeerId); - if ( - currentIntent !== excludedIntent && - currentIntent?.failedParentPeerId === guard.parentPeerId - ) { - return; - } - const connectionKey = viewerConnectionKey(roomId, viewerPeerId); - const failedParents = this.failedParentPeerIdsByViewer.get(connectionKey); - failedParents?.delete(guard.parentPeerId); - if (failedParents?.size === 0) { - this.failedParentPeerIdsByViewer.delete(connectionKey); - } - } - - private pruneHeldQualityParentExclusion( - roomId: string, - viewerPeerId: string, - now: number, - ): void { - if (!Number.isFinite(now)) { - return; - } - const connectionKey = viewerConnectionKey(roomId, viewerPeerId); - const held = this.heldQualityParentExclusionsByViewer.get(connectionKey); - if ( - !held || - (now < held.expiresAtMs && - this.connectedPeer(roomId, held.parentPeerId)?.sessionId === - held.parentSessionId) - ) { - return; - } - this.clearHeldQualityParentExclusion(roomId, viewerPeerId); - } - - private clearHeldQualityParentExclusion( - roomId: string, - viewerPeerId: string, - ): void { - const connectionKey = viewerConnectionKey(roomId, viewerPeerId); - const held = this.heldQualityParentExclusionsByViewer.get(connectionKey); - if (!held) { - return; - } - this.heldQualityParentExclusionsByViewer.delete(connectionKey); - const failedParents = this.failedParentPeerIdsByViewer.get(connectionKey); - failedParents?.delete(held.parentPeerId); - if (failedParents?.size === 0) { - this.failedParentPeerIdsByViewer.delete(connectionKey); - } - } - - private prepareSfuFallback( - roomId: string, - failedPeerId: string, - sourceSessionId: string, - routeIntent: ViewerRouteIntent, - ): SfuPrepareResult { - const fallback = this.options.sfuFallback; - const controller = this.mediaRouteControllers.get(roomId); - const shareGeneration = this.options.getShareGeneration(roomId); - if (!fallback || !controller || !shareGeneration) { - return "unavailable"; - } - if (controller.getPendingRoute()) { - return "busy"; - } - if (this.sfuDisabledRoomIds.has(roomId)) { - this.sendError( - sourceSessionId, - "PEER_NOT_FOUND", - "No media fallback route is available", - ); - return "unavailable"; - } - const active = controller.getActiveRoute(); - const requestedAssignment = active.assignments.get(failedPeerId); - if ( - !requestedAssignment || - requestedAssignment.upstream.kind === "sfu" || - failedPeerId === this.peerRelayTopology.getHostPeerId(roomId) - ) { - this.sendError( - sourceSessionId, - "PEER_NOT_FOUND", - "Viewer has no SFU fallback route", - ); - return "unavailable"; - } - const hostPeerId = this.peerRelayTopology.getHostPeerId(roomId); - if (!hostPeerId) { - this.sendError( - sourceSessionId, - "SERVER_ERROR", - "SFU fallback plan is invalid", - ); - return "unavailable"; - } - - const rootPeerIds = [ - ...new Set([...active.sfu.rootPeerIds, failedPeerId]), - ]; - const assignments = cloneAssignments(active.assignments); - if (!promoteSfuRoots(assignments, rootPeerIds)) { - this.sendError( - sourceSessionId, - "SERVER_ERROR", - "SFU fallback plan is invalid", - ); - return "unavailable"; - } - - let host = assignments.get(hostPeerId); - if (!host) { - this.sendError(sourceSessionId, "SERVER_ERROR", "Media route has no host"); - return "unavailable"; - } - - if ( - !endpointMediaCopyCountFits( - countEndpointMediaCopies({ - childPeerIds: host.childPeerIds, - publicationGeneration: "candidate", - }), - this.endpointMediaCopyCapacity, - ) - ) { - const budgetRootPeerId = - requestedAssignment.upstream.kind === "peer" - ? findHostBranchRoot(active.assignments, failedPeerId, hostPeerId) - : host.childPeerIds[0]; - if (!budgetRootPeerId || budgetRootPeerId === hostPeerId) { - this.sendError( - sourceSessionId, - "PEER_NOT_FOUND", - "No bounded SFU fallback plan is available", - ); - return "unavailable"; - } - if (!rootPeerIds.includes(budgetRootPeerId)) { - rootPeerIds.push(budgetRootPeerId); - } - if (!promoteSfuRoots(assignments, [budgetRootPeerId])) { - this.sendError( - sourceSessionId, - "SERVER_ERROR", - "SFU fallback plan is invalid", - ); - return "unavailable"; - } - host = assignments.get(hostPeerId); - } - - if ( - !host || - !endpointMediaCopyCountFits( - countEndpointMediaCopies({ - childPeerIds: host.childPeerIds, - publicationGeneration: "candidate", - }), - this.endpointMediaCopyCapacity, - ) - ) { - this.sendError( - sourceSessionId, - "PEER_NOT_FOUND", - "Endpoint media copy capacity is exhausted", - ); - return "unavailable"; - } - - const activeHost = active.assignments.get(hostPeerId); - if ( - !activeHost || - !endpointMediaCopyCountFits( - this.activeParentMediaEdges(roomId, hostPeerId, activeHost) + 1, - this.endpointMediaCopyCapacity, - "transition", - ) - ) { - this.sendError( - sourceSessionId, - "PEER_NOT_FOUND", - "Endpoint media copy transition capacity is exhausted", - ); - return "unavailable"; - } - - const generation = randomBytes(16).toString("base64url"); - assignments.set(hostPeerId, { - ...host, - sfuPublicationGeneration: generation, - }); - - const expectedSessionIds = new Map(); - for (const peerId of [hostPeerId, ...rootPeerIds]) { - const peer = this.connectedPeer(roomId, peerId); - if (!peer) { - this.sendError( - sourceSessionId, - "PEER_NOT_FOUND", - "SFU route participant is offline", - ); - return "unavailable"; - } - expectedSessionIds.set(peerId, peer.sessionId); - } - const sfuResourceFence: SfuResourceFence = { - roomId, - shareGeneration, - publicationGeneration: generation, - }; - if (!fallback.admission.reserve(sfuResourceFence, rootPeerIds.length)) { - this.sendError( - sourceSessionId, - "PEER_NOT_FOUND", - "SFU server capacity is exhausted", - ); - return "unavailable"; - } - let revision: number | undefined; - try { - revision = controller.prepare({ - assignments, - expectedParticipantIds: new Set(expectedSessionIds.keys()), - sfuPublicationGeneration: generation, - sfuRootPeerIds: rootPeerIds, - }); - } catch { - this.beginSfuDrain(sfuResourceFence); - this.sendError( - sourceSessionId, - "SERVER_ERROR", - "SFU fallback plan is invalid", - ); - return "unavailable"; - } - if (revision === undefined) { - this.beginSfuDrain(sfuResourceFence); - return "unavailable"; - } - - const timer = setTimeout( - () => this.abortPendingRoute(roomId), - fallback.prepareTimeoutMs ?? DEFAULT_SFU_PREPARE_TIMEOUT_MS, - ); - timer.unref(); - this.pendingRoutePreparations.set(roomId, { - revision, - intentPeerId: failedPeerId, - routeIntent, - qualityIntent: routeIntent.qualityGuard - ? { - guard: { ...routeIntent.qualityGuard }, - takenOverByRouteFailure: false, - } - : null, - expectedSessionIds, - grantsIssued: false, - hostSfuIngressTurnAttempted: false, - healthyReselection: null, - peerQualityReselection: null, - sfuResourceFence, - timer, - }); - void this.issueSfuPrepareGrants( - roomId, - controller, - revision, - sfuResourceFence, - rootPeerIds, - expectedSessionIds, - ); - return "started"; - } - - private async issueSfuPrepareGrants( - roomId: string, - controller: MediaRouteController, - revision: number, - sfuResourceFence: SfuResourceFence, - rootPeerIds: readonly string[], - expectedSessionIds: ReadonlyMap, - ): Promise { - const fallback = this.options.sfuFallback; - if (!fallback) { - return; - } - try { - await fallback.roomControl.createRoom(sfuResourceFence); - let pending = this.pendingRoutePreparations.get(roomId); - let pendingRoute = controller.getPendingRoute(); - if ( - this.mediaRouteControllers.get(roomId) !== controller || - pending?.revision !== revision || - pendingRoute?.route.revision !== revision - ) { - this.beginSfuDrain(sfuResourceFence); - return; - } - if (!this.pendingSessionsAreCurrent(roomId, pending)) { - this.abortPendingRoute(roomId); - return; - } - if (this.abortStalePendingQualityIntent(roomId, pending)) { - return; - } - const grants = await Promise.all( - [...expectedSessionIds].map(async ([peerId, sessionId]) => ({ - peerId, - sessionId, - token: await fallback.tokenIssuer.issueToken({ - roomId, - role: - peerId === this.peerRelayTopology.getHostPeerId(roomId) - ? "host" - : "viewer", - peerId, - shareGeneration: sfuResourceFence.shareGeneration, - publicationGeneration: sfuResourceFence.publicationGeneration, - allowlistedRootPeerIds: rootPeerIds, - }), - })), - ); - pending = this.pendingRoutePreparations.get(roomId); - pendingRoute = controller.getPendingRoute(); - if ( - this.mediaRouteControllers.get(roomId) !== controller || - pending?.revision !== revision || - pendingRoute?.route.revision !== revision - ) { - this.beginSfuDrain(sfuResourceFence); - return; - } - if (!this.pendingSessionsAreCurrent(roomId, pending)) { - this.abortPendingRoute(roomId); - return; - } - if (this.abortStalePendingQualityIntent(roomId, pending)) { - return; - } - for (const { peerId, sessionId, token } of grants) { - const assignment = pendingRoute.route.assignments.get(peerId); - if (!assignment) { - this.abortPendingRoute(roomId); - return; + if (participant.peerId !== room.hostPeerId) { + const edge = snapshot.upstreamByViewer.get(participant.peerId); + if (edge && this.pathIsPhysical(roomId, snapshot, participant.peerId)) { + this.options.setConnectionId(roomId, participant.peerId, edge.connectionId); + } else { + this.options.deleteConnectionId(roomId, participant.peerId); } - this.options.sendToSession(sessionId, { - type: "route-update", - revision, - phase: "prepare", - assignment, - }); - this.options.sendToSession(sessionId, { - type: "sfu-config", - revision, - url: fallback.url, - token, - }); } - pending.grantsIssued = true; - } catch { - console.error("SFU route preparation failed"); - const pending = this.pendingRoutePreparations.get(roomId); - if ( - this.mediaRouteControllers.get(roomId) === controller && - pending?.revision === revision - ) { - this.abortPendingRoute(roomId); - } else { - this.beginSfuDrain(sfuResourceFence); - } - } - } - - private commitPendingRoute(roomId: string, revision: number): void { - const controller = this.mediaRouteControllers.get(roomId); - const pending = this.pendingRoutePreparations.get(roomId); - if ( - !controller || - !pending || - pending.revision !== revision || - !this.pendingSessionsAreCurrent(roomId, pending) - ) { - if (pending?.revision === revision) { - this.abortPendingRoute(roomId); - } - return; - } - if (this.abortStalePendingQualityIntent(roomId, pending)) { - return; - } - const before = controller.getActiveRoute(); - const qualityProbe = pending.peerQualityReselection; - const qualityGuard = pending.qualityIntent?.guard; - if (qualityProbe) { - if ( - !qualityGuard || - qualityProbe.newConnectionId === null || - !this.peerQualityReselectionIsCurrent(roomId, pending) || - !this.peerRelayTopology.reassignViewer( - roomId, - pending.intentPeerId, - this.connectedPeerIds(roomId), - qualityProbe.excludedParentPeerIds, - MAX_PEER_RELAY_DEPTH, - qualityProbe.newParentPeerId, - ) - ) { - this.abortPendingRoute(roomId); - return; - } - } - if (!controller.commit(revision)) { - return; - } - let supersededSfuFences: readonly SfuResourceFence[] = []; - if (pending.sfuResourceFence) { - const result = this.options.sfuFallback?.admission.commit( - pending.sfuResourceFence, - ); - if (!result) { - throw new Error("Committed SFU route has no resource reservation"); - } - supersededSfuFences = result; - } - if ( - pending.hostSfuIngressTurnAttempted && - !this.promoteSelectedSfuIngressAttempt(roomId, revision) - ) { - throw new Error("Committed SFU route has no selected ingress candidate"); - } - clearTimeout(pending.timer); - this.pendingRoutePreparations.delete(roomId); - const active = controller.getActiveRoute(); - const healthy = pending.healthyReselection; - if (healthy) { - this.failedParentPeerIdsByViewer.delete( - viewerConnectionKey(roomId, healthy.rootPeerId), - ); - this.viewerRouteIntentsByRoom.get(roomId)?.delete(healthy.rootPeerId); - this.consumedSfuRefreshesByRoom.delete(roomId); - } else if (qualityProbe?.newConnectionId) { - this.options.setConnectionId( - roomId, - pending.intentPeerId, - qualityProbe.newConnectionId, - ); - this.discardQualityIntent( - roomId, - pending.intentPeerId, - pending.routeIntent, - qualityGuard?.kind === "relative-fps", - ); - } else { - this.clearChangedRouteConnectionIds(roomId, before, active); - } - this.broadcastActiveRoute(roomId, active); - for (const fence of supersededSfuFences) { - this.scheduleSfuDrain(fence); - } - this.drainViewerRouteIntents(roomId); - } - - private healthySfuReselectionIsCurrent( - roomId: string, - pending: PendingRoutePreparation, - ): boolean { - const probe = pending.healthyReselection; - const controller = this.mediaRouteControllers.get(roomId); - const active = controller?.getActiveRoute(); - return ( - probe !== null && - this.pendingRoutePreparations.get(roomId) === pending && - active?.revision === probe.activeRevision && - active.sfu.publicationGeneration === probe.publicationGeneration && - controller?.getPendingRoute()?.route.revision === pending.revision && - this.options.getShareGeneration(roomId) === probe.shareGeneration && - this.pendingSessionsAreCurrent(roomId, pending) - ); - } - - private peerQualityReselectionIsCurrent( - roomId: string, - pending: PendingRoutePreparation, - ): boolean { - const probe = pending.peerQualityReselection; - const controller = this.mediaRouteControllers.get(roomId); - const active = controller?.getActiveRoute(); - const planned = controller?.getPendingRoute()?.route; - const plannedUpstream = planned?.assignments.get( - pending.intentPeerId, - )?.upstream; - const parentAssignment = active?.assignments.get( - probe?.newParentPeerId ?? "", - ); - const hostPeerId = this.peerRelayTopology.getHostPeerId(roomId); - const supportedParent = - probe?.newParentPeerId === hostPeerId - ? parentAssignment?.upstream.kind === "none" - : parentAssignment?.upstream.kind === "peer"; - return Boolean( - probe && - this.pendingRoutePreparations.get(roomId) === pending && - active?.revision === probe.activeRevision && - this.options.getShareGeneration(roomId) === probe.shareGeneration && - supportedParent && - parentAssignment && - endpointMediaCopyCountFits( - this.activeParentMediaEdges( - roomId, - probe.newParentPeerId, - parentAssignment, - ) + 1, - this.peerRelayTopology.getDownstreamCapacity( - roomId, - probe.newParentPeerId, - ), - "transition", - ) && - planned?.revision === pending.revision && - plannedUpstream?.kind === "peer" && - plannedUpstream.peerId === probe.newParentPeerId && - this.pendingSessionsAreCurrent(roomId, pending) && - this.pendingQualityIntentIsCurrent(roomId, pending), - ); - } - - private activeParentMediaEdges( - roomId: string, - parentPeerId: string, - assignment: ParticipantRouteAssignment, - additionalSelectedChildPeerIds: readonly string[] = [], - ): number { - const selectedChildPeerIds = [ - ...(this.selectedPeerEdgeLeasesByRoom.get(roomId)?.values() ?? []), - ] - .filter((lease) => lease.parentPeerId === parentPeerId) - .map((lease) => lease.viewerPeerId); - return countEndpointMediaCopies({ - childPeerIds: assignment.childPeerIds, - publicationGeneration: assignment.sfuPublicationGeneration, - selectedChildPeerIds: [ - ...selectedChildPeerIds, - ...additionalSelectedChildPeerIds, - ], - }); - } - - private ownsActiveSfuForHealthyReselection( - participant: AuthenticatedRouteParticipant, - pending: PendingRoutePreparation, - ): boolean { - const probe = pending.healthyReselection; - if ( - !probe || - pending.expectedSessionIds.get(participant.peerId) !== - participant.sessionId || - !this.healthySfuReselectionIsCurrent(participant.roomId, pending) - ) { - return false; - } - const active = this.mediaRouteControllers - .get(participant.roomId) - ?.getActiveRoute(); - const assignment = active?.assignments.get(participant.peerId); - if (participant.role === "viewer") { - return ( - participant.peerId === probe.rootPeerId && - assignment?.upstream.kind === "sfu" && - active?.sfu.rootPeerIds.includes(participant.peerId) === true - ); - } - return ( - participant.peerId === - this.peerRelayTopology.getHostPeerId(participant.roomId) && - assignment?.sfuPublicationGeneration === probe.publicationGeneration - ); - } - - private ownsActiveOrdinarySfuForPeerQualityReselection( - participant: AuthenticatedRouteParticipant, - pending: PendingRoutePreparation, - ): boolean { - const probe = pending.peerQualityReselection; - if ( - !probe || - participant.peerId !== probe.newParentPeerId || - pending.expectedSessionIds.get(participant.peerId) !== - participant.sessionId || - !this.peerQualityReselectionIsCurrent(participant.roomId, pending) - ) { - return false; - } - const active = this.mediaRouteControllers - .get(participant.roomId) - ?.getActiveRoute(); - const assignment = active?.assignments.get(participant.peerId); - if (!active?.sfu.publicationGeneration || !assignment) { - return false; - } - if (participant.role === "viewer") { - return ( - !this.selectedPeerEdgeLease(participant.roomId, participant.peerId) && - assignment.upstream.kind === "sfu" && - active.sfu.rootPeerIds.includes(participant.peerId) - ); - } - const activeIngress = this.activeSelectedSfuIngresses.get( - participant.roomId, - ); - const hasCurrentSelectedIngress = - activeIngress?.hostPeerId === participant.peerId && - activeIngress.hostSessionId === participant.sessionId && - activeIngress.shareGeneration === probe.shareGeneration && - activeIngress.publicationGeneration === active.sfu.publicationGeneration; - return ( - !hasCurrentSelectedIngress && - participant.peerId === - this.peerRelayTopology.getHostPeerId(participant.roomId) && - assignment.sfuPublicationGeneration === active.sfu.publicationGeneration - ); - } - - private abortPendingRoute( - roomId: string, - excludedPeerId?: string, - drainFailureIntents = true, - ): void { - const controller = this.mediaRouteControllers.get(roomId); - const pending = this.pendingRoutePreparations.get(roomId); - if (!pending) { - return; - } - clearTimeout(pending.timer); - this.pendingRoutePreparations.delete(roomId); - if (pending.sfuResourceFence) { - this.beginSfuDrain(pending.sfuResourceFence); - } - if (pending.healthyReselection) { - this.options.deleteConnectionId(roomId, pending.healthyReselection.rootPeerId); - } - const ingress = this.selectedSfuIngressAttempts.get(roomId); - if (ingress?.revision === pending.revision) { - this.settleSelectedSfuIngressAttempt(roomId); - } - if (controller?.abort(pending.revision)) { - this.broadcastActiveRoute( - roomId, - controller.getActiveRoute(), - excludedPeerId, - ); - } - if (pending.peerQualityReselection) { - this.discardQualityIntent( - roomId, - pending.intentPeerId, - pending.routeIntent, - true, - ); - } - if (drainFailureIntents) { - this.drainViewerRouteIntents(roomId); - } - } - - private abortPendingRouteForTopologyChange( - roomId: string, - excludedPeerId?: string, - ): void { - const pending = this.pendingRoutePreparations.get(roomId); - const currentIntent = pending - ? this.viewerRouteIntentsByRoom - .get(roomId) - ?.get(pending.intentPeerId) - : undefined; - if (pending && currentIntent === pending.routeIntent) { - pending.routeIntent.sfuAttempts = 0; - pending.routeIntent.unavailableReported = false; - } - this.abortPendingRoute(roomId, excludedPeerId, false); - } - - private pendingSessionsAreCurrent( - roomId: string, - pending: PendingRoutePreparation, - ): boolean { - for (const [peerId, sessionId] of pending.expectedSessionIds) { - if (this.connectedPeer(roomId, peerId)?.sessionId !== sessionId) { - return false; - } - } - return true; - } - - private abortStalePendingQualityIntent( - roomId: string, - pending: PendingRoutePreparation, - ): boolean { - if (this.pendingQualityIntentIsCurrent(roomId, pending)) { - return false; - } - this.abortPendingRoute(roomId, undefined, false); - const qualityIntent = pending.qualityIntent; - if (qualityIntent && !qualityIntent.takenOverByRouteFailure) { - const currentIntent = this.viewerRouteIntentsByRoom - .get(roomId) - ?.get(pending.intentPeerId); - if ( - currentIntent === pending.routeIntent && - currentIntent.qualityGuard - ) { - this.discardQualityIntent( - roomId, - pending.intentPeerId, - currentIntent, - ); - } else { - this.releaseQualityOwnedParentExclusion( - roomId, - pending.intentPeerId, - pending.routeIntent, - qualityIntent.guard, - ); - } - } - this.drainViewerRouteIntents(roomId); - return true; - } - - private pendingQualityIntentIsCurrent( - roomId: string, - pending: PendingRoutePreparation, - ): boolean { - const qualityIntent = pending.qualityIntent; - if (!qualityIntent) { - return true; - } - const currentIntent = this.viewerRouteIntentsByRoom - .get(roomId) - ?.get(pending.intentPeerId); - if (currentIntent !== pending.routeIntent) { - return false; - } - if (qualityIntent.takenOverByRouteFailure) { - return ( - currentIntent.qualityGuard === undefined && - currentIntent.failedParentPeerId === qualityIntent.guard.parentPeerId - ); - } - const currentGuard = currentIntent.qualityGuard; - return ( - currentGuard !== undefined && - sameQualityRouteIntentGuard(currentGuard, qualityIntent.guard) && - this.qualityIntentIsCurrent(roomId, pending.intentPeerId, currentIntent) - ); - } - - private async sendFreshSfuConfig( - roomId: string, - peerId: string, - sessionId: string, - revision: number, - reportError = false, - consumeRefresh = false, - ): Promise { - const fallback = this.options.sfuFallback; - const controller = this.mediaRouteControllers.get(roomId); - const active = controller?.getActiveRoute(); - const shareGeneration = this.options.getShareGeneration(roomId); - if ( - !fallback || - !controller || - !active || - active.revision !== revision || - !active.sfu.publicationGeneration || - !shareGeneration || - (peerId !== this.peerRelayTopology.getHostPeerId(roomId) && - !active.sfu.rootPeerIds.includes(peerId)) - ) { - if (reportError) { - this.sendError( - sessionId, - "FORBIDDEN", - "Participant has no active SFU grant", - ); - } - return; - } - if (consumeRefresh) { - let consumed = this.consumedSfuRefreshesByRoom.get(roomId); - if (!consumed) { - consumed = new Set(); - this.consumedSfuRefreshesByRoom.set(roomId, consumed); - } - const refreshKey = sfuRefreshKey( - active.sfu.publicationGeneration, - peerId, - sessionId, - ); - if (consumed.has(refreshKey)) { - this.sendError( - sessionId, - "FORBIDDEN", - "The active SFU grant was already refreshed", - ); - return; - } - consumed.add(refreshKey); - const activeIngress = this.activeSelectedSfuIngresses.get(roomId); - if ( - activeIngress?.hostPeerId === peerId && - activeIngress.hostSessionId === sessionId && - activeIngress.publicationGeneration === - active.sfu.publicationGeneration && - activeIngress.shareGeneration === - this.options.getShareGeneration(roomId) - ) { - this.settleActiveSelectedSfuIngress(roomId); - } - } - try { - const token = await fallback.tokenIssuer.issueToken({ - roomId, - role: - peerId === this.peerRelayTopology.getHostPeerId(roomId) - ? "host" - : "viewer", - peerId, - shareGeneration, - publicationGeneration: active.sfu.publicationGeneration, - allowlistedRootPeerIds: active.sfu.rootPeerIds, - }); - const current = controller.getActiveRoute(); - if ( - this.mediaRouteControllers.get(roomId) !== controller || - this.connectedPeer(roomId, peerId)?.sessionId !== sessionId || - current.revision !== revision || - current.sfu.publicationGeneration !== - active.sfu.publicationGeneration || - this.options.getShareGeneration(roomId) !== shareGeneration - ) { - return; - } - this.options.sendToSession(sessionId, { - type: "sfu-config", - revision, - url: fallback.url, - token, - }); - } catch { - console.error("SFU token refresh failed"); - const current = controller.getActiveRoute(); - if ( - this.mediaRouteControllers.get(roomId) === controller && - this.connectedPeer(roomId, peerId)?.sessionId === sessionId && - current.revision === revision && - current.sfu.publicationGeneration === - active.sfu.publicationGeneration && - this.options.getShareGeneration(roomId) === shareGeneration - ) { - if ( - active.sfu.rootPeerIds.includes(peerId) && - this.startSelectedEdgeTurn(roomId, peerId, true) - ) { - return; - } - this.failBackToPeerBaseline(roomId); - } - } - } - - private reconcileMediaRoute( - roomId: string, - excludedPeerId?: string, - ): MediaRouteController | undefined { - const hostPeerId = this.peerRelayTopology.getHostPeerId(roomId); - if (!hostPeerId) { - return undefined; - } - - let controller = this.mediaRouteControllers.get(roomId); - if (controller?.getPendingRoute()) { - this.abortPendingRouteForTopologyChange(roomId, excludedPeerId); - controller = this.mediaRouteControllers.get(roomId); - } - let initialRevision = 0; - if (controller && !controller.getActiveRoute().assignments.has(hostPeerId)) { - initialRevision = controller.getActiveRoute().revision; - if (initialRevision >= MAX_MEDIA_ROUTE_REVISION) { - throw new Error("Media route revision space exhausted"); - } - initialRevision += 1; - this.clearRoomMediaRouteState(roomId); - controller = undefined; - } - const previousSfu = controller?.getActiveRoute().sfu; - let topologyAssignments = this.peerRelayTopology.getAssignments(roomId); - let rootPeerIds = - previousSfu?.rootPeerIds.filter((peerId) => - topologyAssignments.has(peerId), - ) ?? []; - if ( - previousSfu?.publicationGeneration && - rootPeerIds.length !== previousSfu.rootPeerIds.length - ) { - this.retireShrinkingSfuRoute(roomId, rootPeerIds); - return controller; - } - const assignments = participantRoutesFromTopology( - topologyAssignments, - hostPeerId, - previousSfu?.publicationGeneration ?? null, - rootPeerIds, - this.endpointMediaCopyCapacity, - ); - if (!controller && initialRevision === 0 && assignments.size > 1) { - initialRevision = 1; - } - - if (!controller) { - controller = new MediaRouteController({ - hostPeerId, - endpointMediaCopyCapacity: this.endpointMediaCopyCapacity, - assignments, - revision: initialRevision, - sfuPublicationGeneration: - assignments.get(hostPeerId)?.sfuPublicationGeneration ?? null, - sfuRootPeerIds: [...assignments] - .filter(([, assignment]) => assignment.upstream.kind === "sfu") - .map(([peerId]) => peerId), - }); - this.mediaRouteControllers.set(roomId, controller); - this.broadcastActiveRoute( - roomId, - controller.getActiveRoute(), - excludedPeerId, - ); - return controller; - } - - const publicationGeneration = - rootPeerIds.length > 0 - ? (previousSfu?.publicationGeneration ?? null) - : null; - const revision = controller.reconcileBaseline({ - assignments, - sfuPublicationGeneration: publicationGeneration, - sfuRootPeerIds: rootPeerIds, - }); - if (revision !== undefined) { - this.broadcastActiveRoute( - roomId, - controller.getActiveRoute(), - excludedPeerId, - ); - } else { - this.reconcileSelectedPeerEdgeLeases( - roomId, - controller.getActiveRoute(), - ); - } - return controller; - } - - private broadcastActiveRoute( - roomId: string, - route: RoomMediaRoute, - excludedPeerId?: string, - ): void { - if (route.sfu.publicationGeneration === null) { - this.beginSfuRoomDrain(roomId); - } - this.reconcileSelectedPeerEdgeLeases(roomId, route); - for (const [key, attempt] of this.selectedSfuIngressAttempts) { - if (attempt.roomId === roomId && attempt.revision !== route.revision) { - this.settleSelectedSfuIngressAttempt(key); - } - } - const activeIngress = this.activeSelectedSfuIngresses.get(roomId); - if ( - activeIngress && - (route.sfu.publicationGeneration !== - activeIngress.publicationGeneration || - route.assignments.get(activeIngress.hostPeerId) - ?.sfuPublicationGeneration !== activeIngress.publicationGeneration || - this.connectedPeer(roomId, activeIngress.hostPeerId)?.sessionId !== - activeIngress.hostSessionId || - this.options.getShareGeneration(roomId) !== - activeIngress.shareGeneration) - ) { - this.settleActiveSelectedSfuIngress(roomId); - } - this.clearRoomViewerQualityEvidenceStates(roomId); - for (const [peerId, assignment] of route.assignments) { - if (peerId === excludedPeerId) { - continue; - } - this.sendActiveRouteUpdate(roomId, peerId, route, assignment); } this.options.onActiveRouteChanged?.(roomId); + for (const viewer of this.options.roomStore.getConnectedViewers(roomId)) { + const edge = snapshot.upstreamByViewer.get(viewer.peerId); + if ( + edge?.kind === "sfu" && + edge.physicalActive && + this.pathIsPhysical(roomId, snapshot, viewer.peerId) + ) { + this.options.onViewerMediaState?.( + roomId, + viewer.peerId, + viewer.sessionId, + snapshot.revision, + edge.publicationGeneration, + true, + ); + } + } } - private selectedPeerEdgeLease( + private pathIsPhysical( roomId: string, - viewerPeerId: string, - ): SelectedPeerEdgeLease | undefined { - return this.selectedPeerEdgeLeasesByRoom.get(roomId)?.get(viewerPeerId); + snapshot: RouteSnapshot, + childPeerId: string, + ): boolean { + const hostPeerId = this.rooms.get(roomId)?.hostPeerId; + if (!hostPeerId) return false; + const seen = new Set(); + let current = childPeerId; + while (current !== hostPeerId) { + if (seen.has(current)) return false; + seen.add(current); + const edge = snapshot.upstreamByViewer.get(current); + if (!edge?.physicalActive) return false; + if (edge.kind === "sfu") { + return Boolean( + snapshot.hostPublication?.physicalActive && + snapshot.hostPublication.generation === edge.publicationGeneration, + ); + } + if (edge.parentPeerId === hostPeerId) return true; + current = edge.parentPeerId; + } + return true; } - private selectedPeerEdgeLeaseForPeers( - roomId: string, + private childForPair( + snapshot: RouteSnapshot, firstPeerId: string, secondPeerId: string, - ): SelectedPeerEdgeLease | undefined { - const firstAsViewer = this.selectedPeerEdgeLease(roomId, firstPeerId); - if (firstAsViewer?.parentPeerId === secondPeerId) { - return firstAsViewer; - } - const secondAsViewer = this.selectedPeerEdgeLease(roomId, secondPeerId); - return secondAsViewer?.parentPeerId === firstPeerId - ? secondAsViewer + ): string | undefined { + const first = snapshot.upstreamByViewer.get(firstPeerId); + if (first?.kind === "peer" && first.parentPeerId === secondPeerId) return firstPeerId; + const second = snapshot.upstreamByViewer.get(secondPeerId); + return second?.kind === "peer" && second.parentPeerId === firstPeerId + ? secondPeerId : undefined; } - private selectedPeerEdgeLeaseForParticipantConnection( - roomId: string, - peerId: string, - connectionId: string, - ): SelectedPeerEdgeLease | undefined { - const viewerLease = this.selectedPeerEdgeLease(roomId, peerId); - if (viewerLease?.newConnectionId === connectionId) { - return viewerLease; - } - return [...(this.selectedPeerEdgeLeasesByRoom.get(roomId)?.values() ?? [])] - .find( - (lease) => - lease.parentPeerId === peerId && - lease.newConnectionId === connectionId, - ); - } - - private storeSelectedPeerEdgeLease(lease: SelectedPeerEdgeLease): void { - let roomLeases = this.selectedPeerEdgeLeasesByRoom.get(lease.roomId); - if (!roomLeases) { - roomLeases = new Map(); - this.selectedPeerEdgeLeasesByRoom.set(lease.roomId, roomLeases); - } - roomLeases.set(lease.viewerPeerId, lease); - } - - private selectedPeerEdgeLeaseIsCurrent( - roomId: string, - lease: SelectedPeerEdgeLease, - route: RoomMediaRoute, - ): boolean { - const viewerAssignment = this.peerRelayTopology.getAssignment( - roomId, - lease.viewerPeerId, - ); - const parentAssignment = this.peerRelayTopology.getAssignment( - roomId, - lease.parentPeerId, - ); - const activeParentAssignment = route.assignments.get(lease.parentPeerId); - const hostPeerId = this.peerRelayTopology.getHostPeerId(roomId); - const parentSupportsOutboundRelay = - lease.parentPeerId === hostPeerId - ? activeParentAssignment?.upstream.kind === "none" - : activeParentAssignment?.upstream.kind === "peer"; - const activeParentEdges = activeParentAssignment - ? this.activeParentMediaEdges( - roomId, - lease.parentPeerId, - activeParentAssignment, - ) - : Number.POSITIVE_INFINITY; - return ( - route.revision === lease.currentRouteRevision && - this.options.getShareGeneration(roomId) === lease.shareGeneration && - this.connectedPeer(roomId, lease.parentPeerId)?.sessionId === - lease.parentSessionId && - this.options.roomStore.getConnectedViewer(roomId, lease.viewerPeerId) - ?.sessionId === lease.viewerSessionId && - parentSupportsOutboundRelay && - viewerAssignment?.parentPeerId === lease.parentPeerId && - parentAssignment?.childPeerIds.includes(lease.viewerPeerId) === true && - endpointMediaCopyCountFits( - activeParentEdges, - this.peerRelayTopology.getAdvertisedDownstreamCapacity( - roomId, - lease.parentPeerId, - ), - ) - ); - } - - private reconcileSelectedPeerEdgeLeases( - roomId: string, - route: RoomMediaRoute, - ): void { - const leases = this.selectedPeerEdgeLeasesByRoom.get(roomId); - if (!leases) { - return; - } - for (const lease of [...leases.values()].reverse()) { - lease.currentRouteRevision = route.revision; - if (!this.selectedPeerEdgeLeaseIsCurrent(roomId, lease, route)) { - this.settleSelectedPeerEdgeLease(roomId, lease.viewerPeerId); - } - } - } - - private settleSelectedPeerEdgeLease( - roomId: string, - viewerPeerId: string, - excludedPeerId?: string, - ): SelectedPeerEdgeLease | undefined { - const roomLeases = this.selectedPeerEdgeLeasesByRoom.get(roomId); - const lease = roomLeases?.get(viewerPeerId); - if (!lease) { - return undefined; - } - clearTimeout(lease.timer); - roomLeases!.delete(viewerPeerId); - if (roomLeases!.size === 0) { - this.selectedPeerEdgeLeasesByRoom.delete(roomId); - } - this.releaseTurnAllocation(lease.allocationFence); - - const active = this.mediaRouteControllers.get(lease.roomId)?.getActiveRoute(); - if (!active) { - return lease; - } - for (const [peerId, sessionId] of [ - [lease.parentPeerId, lease.parentSessionId], - [lease.viewerPeerId, lease.viewerSessionId], - ] as const) { - if (peerId === excludedPeerId) { - continue; - } - const participant = this.connectedPeer(lease.roomId, peerId); - const assignment = active.assignments.get(peerId); - if (participant?.sessionId === sessionId && assignment) { - this.sendActiveRouteUpdate(lease.roomId, peerId, active, assignment); - } - } - return lease; - } - - private sendActiveRouteUpdate( - roomId: string, - peerId: string, - route: RoomMediaRoute, - assignment = route.assignments.get(peerId), - ): void { - const peer = this.connectedPeer(roomId, peerId); - if (!peer || !assignment) { - return; - } - for (const lease of this.selectedPeerEdgeLeasesByRoom - .get(roomId) - ?.values() ?? []) { - if ( - (lease.parentPeerId === peerId || lease.viewerPeerId === peerId) && - this.selectedPeerEdgeLeaseIsCurrent(roomId, lease, route) - ) { - this.options.sendToSession(peer.sessionId, { - ...lease.grant, - revision: route.revision, - }); - } - } - this.options.sendToSession(peer.sessionId, { - type: "route-update", - revision: route.revision, - phase: "active", - assignment, - }); - } - - private clearChangedRouteConnectionIds( - roomId: string, - before: RoomMediaRoute, - after: RoomMediaRoute, - ): void { - for (const [peerId, assignment] of after.assignments) { - const previous = before.assignments.get(peerId); - if (!previous || !sameUpstream(previous, assignment)) { - this.options.deleteConnectionId(roomId, peerId); - } - } - } - - private clearChangedConnectionIds( - roomId: string, - changes: readonly MediaAssignmentChange[], - ): void { - for (const change of changes) { - if ( - change.previousParentPeerId !== undefined && - change.previousParentPeerId !== change.mediaAssignment.parentPeerId - ) { - this.options.deleteConnectionId(roomId, change.peerId); - } - } - } - - private sendMediaAssignmentChanges( - roomId: string, - changes: readonly MediaAssignmentChange[], - excludedPeerId?: string, - ): void { - const active = this.mediaRouteControllers.get(roomId)?.getActiveRoute(); - for (const change of changes) { - if (change.peerId === excludedPeerId) { - continue; - } - const peer = this.connectedPeer(roomId, change.peerId); - if (peer) { - const activeAssignment = active?.assignments.get(change.peerId); - this.options.sendToSession(peer.sessionId, { - type: "media-assignment", - mediaAssignment: activeAssignment - ? mediaAssignmentFromParticipantRoute(activeAssignment) - : change.mediaAssignment, - }); - } - } - } - - private sendCurrentParentAssignment(roomId: string, childPeerId: string): void { - const active = this.mediaRouteControllers.get(roomId)?.getActiveRoute(); - const child = active?.assignments.get(childPeerId); - const parentPeerId = - child?.upstream.kind === "peer" ? child.upstream.peerId : undefined; - if (!parentPeerId) { - return; - } - const parent = this.connectedPeer(roomId, parentPeerId); - const parentAssignment = active?.assignments.get(parentPeerId); - if (parent && parentAssignment && active) { - this.sendActiveRouteUpdate( - roomId, - parentPeerId, - active, - parentAssignment, - ); - this.options.sendToSession(parent.sessionId, { - type: "media-assignment", - mediaAssignment: mediaAssignmentFromParticipantRoute(parentAssignment), - }); - } - } - - private connectedPeerIds(roomId: string): Set { - const peerIds = new Set( - this.options.roomStore - .getConnectedViewers(roomId) - .map((viewer) => viewer.peerId), - ); - const host = this.options.roomStore.getConnectedHost(roomId); - if (host) { - peerIds.add(host.peerId); - } - return peerIds; - } - private connectedPeer(roomId: string, peerId: string) { const host = this.options.roomStore.getConnectedHost(roomId); - if (host?.peerId === peerId) { - return host; - } - return this.options.roomStore.getConnectedViewer(roomId, peerId); + return host?.peerId === peerId + ? host + : this.options.roomStore.getConnectedViewer(roomId, peerId); } - private sendError(sessionId: string, code: ErrorCode, message: string): void { - this.options.sendToSession(sessionId, { type: "error", code, message }); - } - - private clearRoomFailedParents(roomId: string): void { - const prefix = `${roomId}:`; - for (const key of this.failedParentPeerIdsByViewer.keys()) { - if (key.startsWith(prefix)) { - this.failedParentPeerIdsByViewer.delete(key); - } + private commitReservation( + reservation: CandidateReservation, + ): boolean { + if (reservation.kind === "direct") return true; + if (reservation.kind === "selected-turn") { + return reservation.edge.kind === "turn" && + this.options.selectedEdgeTurn?.admission.commit(reservation.edge.fence) === true; } - for (const key of this.heldQualityParentExclusionsByViewer.keys()) { - if (key.startsWith(prefix)) { - this.heldQualityParentExclusionsByViewer.delete(key); - } + if (reservation.kind === "sfu-reuse") { + return reservation.edge.kind === "sfu-subscription" && + this.options.sfuFallback?.admission.commitSubscription( + reservation.edge.fence, + ) === true; } - } - - private beginSfuDrain(fence: SfuResourceFence): boolean { - const fallback = this.options.sfuFallback; - if (!fallback || !fallback.admission.beginDrain(fence)) { + if (reservation.publication.kind !== "sfu-publication") return false; + if ( + reservation.publication.ingressTurn && + this.options.selectedEdgeTurn?.admission.commit( + reservation.publication.ingressTurn.fence, + ) !== true + ) { return false; } - this.scheduleSfuDrain(fence); + const draining = this.options.sfuFallback?.admission.commitPublication( + reservation.publication.fence, + ); + if (!draining) return false; + for (const fence of draining) this.scheduleSfuDrain(fence); return true; } - private beginSfuRoomDrain(roomId: string): void { - const fallback = this.options.sfuFallback; - if (!fallback) { + private releaseReservation(reservation: CandidateReservation): void { + this.releaseResources(reservationResources(reservation)); + } + + private releaseResources(resources: readonly RouteResource[]): void { + for (const resource of resources) this.releaseResource(resource); + } + + private releaseResource(resource: RouteResource): void { + if (resource.released) return; + resource.released = true; + if (resource.kind === "overlap") return; + if (resource.kind === "turn") { + const admission = this.options.selectedEdgeTurn?.admission; + if (admission?.beginDrain(resource.fence)) { + admission.completeDrain(resource.fence); + this.wakeResourceWaiters(); + } return; } - for (const fence of fallback.admission.beginDrainRoom(roomId)) { - this.scheduleSfuDrain(fence); + if (resource.kind === "sfu-subscription") { + this.options.sfuFallback?.admission.releaseSubscription(resource.fence); + return; + } + if (resource.ingressTurn) this.releaseResource(resource.ingressTurn); + if (this.options.sfuFallback?.admission.beginDrain(resource.fence)) { + this.scheduleSfuDrain(resource.fence); } } - private scheduleSfuDrain(fence: SfuResourceFence): void { + private wakeResourceWaiters(): void { + const roomIds = [...this.resourceWaiterRoomIds]; + this.resourceWaiterRoomIds.clear(); + for (const roomId of roomIds) { + const controller = this.rooms.get(roomId)?.controller; + if (!controller) continue; + controller.touchExternalFacts(); + this.requestPump(roomId); + } + } + + private scheduleDeadline( + roomId: string, + room: RoomRuntime, + operation: OperationSnapshot, + ): void { + this.clearDeadline(room); + const timer = setTimeout(() => { + if (room.deadlineTimer !== timer || !room.controller) return; + room.deadlineTimer = undefined; + const before = room.controller.snapshot().revision; + const expired = room.controller.operationExpired(this.now()); + this.releaseResources(expired.released); + if (room.controller.snapshot().revision !== before) this.broadcastActive(roomId, room); + if (expired.exhausted) { + this.sendViewerError( + roomId, + operation.childPeerId, + "PEER_NOT_FOUND", + "No usable media route is available", + ); + } + this.requestPump(roomId); + }, Math.max(0, operation.deadlineAtMs - this.now())); + timer.unref(); + room.deadlineTimer = timer; + } + + private clearDeadline(room: RoomRuntime): void { + if (room.deadlineTimer) clearTimeout(room.deadlineTimer); + room.deadlineTimer = undefined; + } + + private async sendFreshSfuConfig( + participant: AuthenticatedRouteParticipant, + ): Promise { const fallback = this.options.sfuFallback; - if (!fallback) { + const room = this.rooms.get(participant.roomId); + const snapshot = room?.controller?.snapshot(); + const publication = snapshot?.hostPublication; + const shareGeneration = this.options.getShareGeneration(participant.roomId); + if (!fallback || !room || !snapshot || !publication?.physicalActive || !shareGeneration) { return; } + const isHost = participant.peerId === room.hostPeerId; + const edge = snapshot.upstreamByViewer.get(participant.peerId); + if (!isHost && (edge?.kind !== "sfu" || !edge.physicalActive)) return; + const token = await fallback.tokenIssuer.issueToken({ + roomId: participant.roomId, + role: isHost ? "host" : "viewer", + peerId: participant.peerId, + shareGeneration, + publicationGeneration: publication.generation, + allowlistedRootPeerIds: isHost + ? [...snapshot.upstreamByViewer] + .filter(([, candidate]) => candidate.kind === "sfu" && candidate.physicalActive) + .map(([peerId]) => peerId) + : [participant.peerId], + }); + const current = room.controller?.snapshot(); + if ( + current?.revision !== snapshot.revision || + current.hostPublication?.generation !== publication.generation || + this.connectedPeer(participant.roomId, participant.peerId)?.sessionId !== + participant.sessionId + ) { + return; + } + this.options.sendToSession(participant.sessionId, { + type: "sfu-config", + revision: snapshot.revision, + url: fallback.url, + token, + }); + } + + private activeSfuFence(roomId: string): SfuResourceFence | null { + const publication = this.rooms.get(roomId)?.controller?.snapshot().hostPublication; + return publication?.physicalActive && publication.resource.kind === "sfu-publication" + ? publication.resource.fence + : null; + } + + private scheduleHostOfflineCheck(roomId: string, hostPeerId: string): void { + this.cancelHostOfflineCheck(roomId); + const fallback = this.options.sfuFallback; + const fence = this.activeSfuFence(roomId); + if (!fallback || !fence || this.closing) return; + const timer = setTimeout(() => { + void this.checkHostOffline(roomId).catch(() => undefined); + }, fallback.hostOfflineCheckMs ?? DEFAULT_HOST_OFFLINE_CHECK_MS); + timer.unref(); + this.hostOfflineChecks.set(roomId, { hostPeerId, fence, timer }); + } + + private async checkHostOffline(roomId: string): Promise { + const check = this.hostOfflineChecks.get(roomId); + const fallback = this.options.sfuFallback; + if (!check || !fallback || this.closing) return; + if (this.options.roomStore.getConnectedHost(roomId)) { + this.cancelHostOfflineCheck(roomId); + return; + } + let exists = true; + try { + exists = await fallback.roomControl.hostParticipantExists(check.fence); + } catch { + this.hostOfflineChecks.delete(roomId); + this.scheduleHostOfflineCheck(roomId, check.hostPeerId); + return; + } + if (this.hostOfflineChecks.get(roomId) !== check) return; + this.hostOfflineChecks.delete(roomId); + if (exists) { + this.scheduleHostOfflineCheck(roomId, check.hostPeerId); + return; + } + const room = this.rooms.get(roomId); + const controller = room?.controller; + const snapshot = controller?.snapshot(); + const publication = snapshot?.hostPublication; + if (!room || !controller || !snapshot || !publication) return; + controller.invalidateHostPublication({ + hostSessionId: publication.hostSessionId, + routeRevision: snapshot.revision, + generation: publication.generation, + connectionId: publication.connectionId, + }); + const invalid = controller.snapshot(); + this.releaseResources( + controller.retireHostPublication({ + hostSessionId: publication.hostSessionId, + routeRevision: invalid.revision, + generation: publication.generation, + connectionId: publication.connectionId, + }), + ); + this.broadcastActive(roomId, room); + this.requestPump(roomId); + } + + private cancelHostOfflineCheck(roomId: string): void { + const check = this.hostOfflineChecks.get(roomId); + if (check) clearTimeout(check.timer); + this.hostOfflineChecks.delete(roomId); + } + + private scheduleSfuDrain(fence: SfuResourceFence): void { + if (!this.options.sfuFallback) return; const key = managedSfuRoomName(fence); let task = this.sfuDrainTasks.get(key); if (!task) { @@ -4114,703 +1542,152 @@ export class HybridMediaRouter { } private runSfuDrain(key: string, task: SfuDrainTask): Promise { - if (task.operation) { - return task.operation; - } + if (task.operation) return task.operation; const fallback = this.options.sfuFallback; - if (!fallback) { - return Promise.resolve(); - } + if (!fallback) return Promise.resolve(); let tracked!: Promise; tracked = (async () => { try { await fallback.roomControl.deleteRoom(task.fence); - if (this.sfuDrainTasks.get(key) !== task) { - return; - } + if (this.sfuDrainTasks.get(key) !== task) return; if (!fallback.admission.completeDrain(task.fence)) { throw new Error("LiveKit drain has no matching resource generation"); } - if (task.retryTimer) { - clearTimeout(task.retryTimer); - } this.sfuDrainTasks.delete(key); + this.wakeResourceWaiters(); } catch (error) { - if (this.sfuDrainTasks.get(key) !== task) { - return; - } - if (this.closing) { - throw error; - } - console.error("SFU room drain failed; capacity remains reserved"); - if (!task.retryTimer) { - task.retryTimer = setTimeout(() => { - task.retryTimer = undefined; - void this.runSfuDrain(key, task).catch(() => undefined); - }, fallback.drainRetryMs ?? DEFAULT_SFU_DRAIN_RETRY_MS); - task.retryTimer.unref(); - } + if (this.sfuDrainTasks.get(key) !== task) return; + if (this.closing) throw error; + task.retryTimer = setTimeout(() => { + task.retryTimer = undefined; + void this.runSfuDrain(key, task).catch(() => undefined); + }, fallback.drainRetryMs ?? DEFAULT_SFU_DRAIN_RETRY_MS); + task.retryTimer.unref(); } })().finally(() => { - if (task.operation === tracked) { - task.operation = undefined; - } + if (task.operation === tracked) task.operation = undefined; }); task.operation = tracked; return tracked; } - private activeSfuFence(roomId: string): SfuResourceFence | null { - const publicationGeneration = this.mediaRouteControllers - .get(roomId) - ?.getActiveRoute().sfu.publicationGeneration; - const shareGeneration = this.options.getShareGeneration(roomId); - return publicationGeneration && shareGeneration - ? { roomId, shareGeneration, publicationGeneration } - : null; - } - - private scheduleHostOfflineCheck(roomId: string, hostPeerId: string): void { + private clearRoom(roomId: string): void { + const room = this.rooms.get(roomId); + if (!room) return; + this.clearDeadline(room); this.cancelHostOfflineCheck(roomId); - const fallback = this.options.sfuFallback; - const fence = this.activeSfuFence(roomId); - if (!fallback || !fence || this.closing) { - return; + const controller = room.controller; + room.controller = undefined; + room.requested = false; + if (controller) this.releaseResources(controller.dispose()); + this.rooms.delete(roomId); + this.resourceWaiterRoomIds.delete(roomId); + for (const fence of this.options.sfuFallback?.admission.beginDrainRoom(roomId) ?? []) { + this.scheduleSfuDrain(fence); } - const timer = setTimeout(() => { - void this.checkHostOffline(roomId).catch(() => undefined); - }, fallback.hostOfflineCheckMs ?? DEFAULT_HOST_OFFLINE_CHECK_MS); - timer.unref(); - this.hostOfflineChecks.set(roomId, { - hostPeerId, - fence, - timer, - }); + const turnAdmission = this.options.selectedEdgeTurn?.admission; + for (const fence of turnAdmission?.beginDrainRoom(roomId) ?? []) { + turnAdmission!.completeDrain(fence); + } + this.wakeResourceWaiters(); } - private async checkHostOffline(roomId: string): Promise { - const check = this.hostOfflineChecks.get(roomId); - const fallback = this.options.sfuFallback; - if (!check || !fallback || this.closing) { - return; - } - if (this.options.roomStore.getConnectedHost(roomId)) { - this.cancelHostOfflineCheck(roomId); - return; - } - - let hostExists: boolean; - try { - hostExists = await fallback.roomControl.hostParticipantExists(check.fence); - } catch { - if (this.hostOfflineChecks.get(roomId) === check) { - console.error("SFU Host-offline check failed; capacity remains reserved"); - this.hostOfflineChecks.delete(roomId); - this.scheduleHostOfflineCheck(roomId, check.hostPeerId); - } - return; - } - if (this.hostOfflineChecks.get(roomId) !== check) { - return; - } - if (this.options.roomStore.getConnectedHost(roomId)) { - this.cancelHostOfflineCheck(roomId); - return; - } - const currentFence = this.activeSfuFence(roomId); - if (!currentFence || !sameSfuFence(currentFence, check.fence)) { - this.cancelHostOfflineCheck(roomId); - return; - } - this.hostOfflineChecks.delete(roomId); - if (hostExists) { - this.scheduleHostOfflineCheck(roomId, check.hostPeerId); - return; - } - this.clearRoomMediaRouteState(roomId); + private sendRoomError(roomId: string, code: ErrorCode, message: string): void { + const host = this.options.roomStore.getConnectedHost(roomId); + if (host) this.options.sendToSession(host.sessionId, { type: "error", code, message }); } - private cancelHostOfflineCheck(roomId: string): void { - const check = this.hostOfflineChecks.get(roomId); - if (check) { - clearTimeout(check.timer); - this.hostOfflineChecks.delete(roomId); - } - } - - private clearRoomMediaRouteState(roomId: string): void { - this.cancelHostOfflineCheck(roomId); - const pending = this.pendingRoutePreparations.get(roomId); - if (pending) { - clearTimeout(pending.timer); - this.pendingRoutePreparations.delete(roomId); - } - this.mediaRouteControllers.delete(roomId); - this.clearRoomFailedParents(roomId); - this.viewerRouteIntentsByRoom.delete(roomId); - const selectedLeases = this.selectedPeerEdgeLeasesByRoom.get(roomId); - if (selectedLeases) { - for (const lease of selectedLeases.values()) { - clearTimeout(lease.timer); - this.releaseTurnAllocation(lease.allocationFence); - } - this.selectedPeerEdgeLeasesByRoom.delete(roomId); - } - this.settleSelectedSfuIngressAttempt(roomId); - this.settleActiveSelectedSfuIngress(roomId); - this.clearDeferredHealthySfuReselection(roomId); - this.consumedSfuRefreshesByRoom.delete(roomId); - this.sfuDisabledRoomIds.delete(roomId); - this.clearRoomViewerQualityEvidenceStates(roomId); - this.roomQualityMigrationCooldownUntilMs.delete(roomId); - this.beginSfuRoomDrain(roomId); - } - - private settleSelectedSfuIngressAttempt( + private sendViewerError( roomId: string, - ): SelectedSfuIngressAttempt | undefined { - const attempt = this.selectedSfuIngressAttempts.get(roomId); - if (!attempt) { - return undefined; - } - clearTimeout(attempt.timer); - this.selectedSfuIngressAttempts.delete(roomId); - this.releaseTurnAllocation(attempt.allocationFence); - return attempt; - } - - private promoteSelectedSfuIngressAttempt( - roomId: string, - revision: number, - ): boolean { - const attempt = this.selectedSfuIngressAttempts.get(roomId); - if (!attempt || attempt.revision !== revision || !attempt.answered) { - return false; - } - clearTimeout(attempt.timer); - this.selectedSfuIngressAttempts.delete(roomId); - this.settleActiveSelectedSfuIngress(roomId); - this.activeSelectedSfuIngresses.set(roomId, { - shareGeneration: attempt.shareGeneration, - hostPeerId: attempt.hostPeerId, - hostSessionId: attempt.hostSessionId, - publicationGeneration: attempt.publicationGeneration, - connectionId: attempt.newConnectionId, - allocationFence: attempt.allocationFence, - }); - return true; - } - - private settleActiveSelectedSfuIngress( - roomId: string, - ): ActiveSelectedSfuIngress | undefined { - const ingress = this.activeSelectedSfuIngresses.get(roomId); - if (!ingress) { - return undefined; - } - this.activeSelectedSfuIngresses.delete(roomId); - this.releaseTurnAllocation(ingress.allocationFence); - return ingress; - } - - private releaseTurnAllocation(fence: SelectedEdgeTurnIdentity): boolean { - const admission = this.options.selectedEdgeTurn?.admission; - if (!admission?.beginDrain(fence)) { - return false; - } - if (!admission.completeDrain(fence)) { - throw new Error("TURN allocation drain could not be completed"); - } - return true; - } - - private clearSelectedEdgeTurnsForParticipant( - roomId: string, - peerId: string, + viewerPeerId: string, + code: ErrorCode, + message: string, ): void { - const leases = this.selectedPeerEdgeLeasesByRoom.get(roomId); - for (const lease of [...(leases?.values() ?? [])]) { - if (lease.parentPeerId === peerId || lease.viewerPeerId === peerId) { - this.settleSelectedPeerEdgeLease( - roomId, - lease.viewerPeerId, - peerId, - ); - } - } - const ingress = this.selectedSfuIngressAttempts.get(roomId); - if (ingress?.hostPeerId === peerId) { - this.settleSelectedSfuIngressAttempt(roomId); - } - if (this.activeSelectedSfuIngresses.get(roomId)?.hostPeerId === peerId) { - this.settleActiveSelectedSfuIngress(roomId); - } - } - - private clearSfuRefreshesForPeer(roomId: string, peerId: string): void { - const consumed = this.consumedSfuRefreshesByRoom.get(roomId); - if (!consumed) { - return; - } - const marker = `\u0000${peerId}\u0000`; - for (const key of consumed) { - if (key.includes(marker)) { - consumed.delete(key); - } - } - if (consumed.size === 0) { - this.consumedSfuRefreshesByRoom.delete(roomId); - } - } - - private startRoomQualityMigrationCooldown(roomId: string): void { - const now = this.options.now?.() ?? Date.now(); - if (!Number.isFinite(now)) { - return; - } - this.roomQualityMigrationCooldownUntilMs.set( - roomId, - now + VIEWER_QUALITY_REASSIGN_COOLDOWN_MS, - ); - } - - private deferHealthySfuReselection( - participant: AuthenticatedRouteParticipant, - activeRevision: number, - shareGeneration: string, - publicationGeneration: string, - cooldownUntil: number, - now: number, - ): void { - this.clearDeferredHealthySfuReselection(participant.roomId); - const deferred: DeferredHealthySfuReselection = { - activeRevision, - shareGeneration, - publicationGeneration, - rootPeerId: participant.peerId, - rootSessionId: participant.sessionId, - timer: null, - }; - const schedule = (delayMs: number) => { - deferred.timer = setTimeout(reassert, Math.max(0, delayMs)); - deferred.timer.unref(); - }; - const reassert = () => { - if ( - this.deferredHealthySfuReselections.get(participant.roomId) !== deferred - ) { - return; - } - deferred.timer = null; - const currentTime = this.options.now?.() ?? Date.now(); - if (!Number.isFinite(currentTime)) { - this.deferredHealthySfuReselections.delete(participant.roomId); - return; - } - const currentCooldown = - this.roomQualityMigrationCooldownUntilMs.get(participant.roomId) ?? 0; - if (currentCooldown > currentTime) { - schedule(currentCooldown - currentTime); - return; - } - - this.deferredHealthySfuReselections.delete(participant.roomId); - const viewer = this.options.roomStore.getConnectedViewer( - participant.roomId, - deferred.rootPeerId, - ); - const controller = this.mediaRouteControllers.get(participant.roomId); - const active = controller?.getActiveRoute(); - const assignment = active?.assignments.get(deferred.rootPeerId); - if ( - viewer?.sessionId !== deferred.rootSessionId || - !controller || - controller.getPendingRoute() || - this.roomPeerMigrationAttempt( - participant.roomId, - participant.peerId, - ) !== null || - active?.revision !== deferred.activeRevision || - active.sfu.publicationGeneration !== deferred.publicationGeneration || - active.sfu.rootPeerIds.length !== 1 || - active.sfu.rootPeerIds[0] !== deferred.rootPeerId || - assignment?.upstream.kind !== "sfu" || - this.options.getShareGeneration(participant.roomId) !== - deferred.shareGeneration - ) { - return; - } - - // Duplicate authority asks the Viewer for fresh media proof. - this.sendActiveRouteUpdate( - participant.roomId, - deferred.rootPeerId, - active, - assignment, - ); - }; - this.deferredHealthySfuReselections.set(participant.roomId, deferred); - schedule(cooldownUntil - now); - } - - private clearDeferredHealthySfuReselection(roomId: string): void { - const deferred = this.deferredHealthySfuReselections.get(roomId); - if (!deferred) { - return; - } - if (deferred.timer) { - clearTimeout(deferred.timer); - } - this.deferredHealthySfuReselections.delete(roomId); - } - - private clearDeferredHealthySfuReselectionForParticipant( - roomId: string, - peerId: string, - ): void { - if ( - this.deferredHealthySfuReselections.get(roomId)?.rootPeerId === peerId - ) { - this.clearDeferredHealthySfuReselection(roomId); - } - } - - private clearViewerQualityStateForParticipant( - roomId: string, - peerId: string, - ): void { - this.clearHeldQualityParentExclusion(roomId, peerId); - const intents = this.viewerRouteIntentsByRoom.get(roomId); - for (const [viewerPeerId, intent] of [...(intents ?? [])]) { - if ( - intent.qualityGuard && - (viewerPeerId === peerId || intent.qualityGuard.parentPeerId === peerId) - ) { - this.discardQualityIntent(roomId, viewerPeerId, intent); - } - } - const connectionKey = viewerConnectionKey(roomId, peerId); - this.viewerQualityEvidenceStates.delete(connectionKey); - for (const [key, state] of this.viewerQualityEvidenceStates) { - if (key.startsWith(`${roomId}:`) && state.parentPeerId === peerId) { - this.viewerQualityEvidenceStates.delete(key); - } - } - } - - private clearRoomViewerQualityEvidenceStates(roomId: string): void { - const prefix = `${roomId}:`; - for (const key of this.viewerQualityEvidenceStates.keys()) { - if (key.startsWith(prefix)) { - this.viewerQualityEvidenceStates.delete(key); - } - } - } - - private enqueueUnassignedViewerIntents(roomId: string): void { - const advertised = this.relayCapacitySessionsByRoom.get(roomId); - const assignments = this.mediaRouteControllers - .get(roomId) - ?.getActiveRoute().assignments; - if (!advertised || !assignments) { - return; - } - let intents = this.viewerRouteIntentsByRoom.get(roomId); - for (const [peerId, sessionId] of advertised) { - const viewer = this.options.roomStore.getConnectedViewer(roomId, peerId); - if (viewer?.sessionId !== sessionId) { - advertised.delete(peerId); - intents?.delete(peerId); - continue; - } - if (assignments.get(peerId)?.upstream.kind !== "none") { - continue; - } - if (!intents) { - intents = new Map(); - this.viewerRouteIntentsByRoom.set(roomId, intents); - } - const existing = intents.get(peerId); - if ( - existing?.sessionId === sessionId && - existing.failedParentPeerId === null - ) { - continue; - } - intents.set(peerId, { - failedParentPeerId: null, - sessionId, - sfuAttempts: 0, - unavailableReported: false, - }); - } + const viewer = this.options.roomStore.getConnectedViewer(roomId, viewerPeerId); + if (viewer) this.options.sendToSession(viewer.sessionId, { type: "error", code, message }); } } -function viewerConnectionKey(roomId: string, peerId: string): string { - return `${roomId}:${peerId}`; +function cursorGuard( + operation: OperationSnapshot, + plan: CandidatePlan, +): CandidateCursorGuard { + return { + childPeerId: operation.childPeerId, + childSessionId: operation.childSessionId, + baseRevision: operation.baseRevision, + factVersion: operation.factVersion, + cursor: operation.cursor, + plan, + }; } -function sameSfuFence(left: SfuResourceFence, right: SfuResourceFence): boolean { - return ( - left.roomId === right.roomId && - left.shareGeneration === right.shareGeneration && - left.publicationGeneration === right.publicationGeneration - ); +function emptyAssignment(): ParticipantRouteAssignment { + return { + upstream: { kind: "none" }, + childPeerIds: [], + sfuPublicationGeneration: null, + }; } -function sameQualityRouteIntentGuard( - left: QualityRouteIntentGuard, - right: QualityRouteIntentGuard, -): boolean { - return ( - left.kind === right.kind && - left.viewerSessionId === right.viewerSessionId && - left.connectionId === right.connectionId && - left.routeRevision === right.routeRevision && - left.parentPeerId === right.parentPeerId && - left.parentSessionId === right.parentSessionId && - left.exclusionOwnedByQuality === right.exclusionOwnedByQuality - ); -} - -function sfuRefreshKey( - publicationGeneration: string, - peerId: string, - sessionId: string, -): string { - return `${publicationGeneration}\u0000${peerId}\u0000${sessionId}`; -} - -function participantRouteFromMediaAssignment( - assignment: MediaAssignment | undefined, -): ParticipantRouteAssignment | undefined { - return assignment - ? { - upstream: assignment.parentPeerId - ? { kind: "peer", peerId: assignment.parentPeerId } - : { kind: "none" }, - childPeerIds: [...assignment.childPeerIds], - sfuPublicationGeneration: null, - } - : undefined; -} - -function mediaAssignmentFromParticipantRoute( - assignment: ParticipantRouteAssignment, -): MediaAssignment { +function mediaAssignment(assignment: ParticipantRouteAssignment): MediaAssignment { return { parentPeerId: - assignment.upstream.kind === "peer" - ? assignment.upstream.peerId - : null, + assignment.upstream.kind === "peer" ? assignment.upstream.peerId : null, childPeerIds: [...assignment.childPeerIds], }; } -function participantRoutesFromTopology( - topologyAssignments: ReadonlyMap, - hostPeerId: string, - publicationGeneration: string | null, - requestedRootPeerIds: readonly string[], - endpointMediaCopyCapacity: number, -): Map { - // The topology remains the failback baseline. The active SFU overlay detaches - // children because an SFU-fed Viewer has no independent outbound proof yet. - const assignments = new Map(); - for (const [peerId, assignment] of topologyAssignments) { - assignments.set(peerId, { - upstream: - peerId === hostPeerId || !assignment.parentPeerId - ? { kind: "none" } - : { kind: "peer", peerId: assignment.parentPeerId }, - childPeerIds: [...assignment.childPeerIds], - sfuPublicationGeneration: null, - }); - } - - const rootPeerIds = publicationGeneration - ? requestedRootPeerIds.filter( - (peerId) => peerId !== hostPeerId && assignments.has(peerId), - ) - : []; - for (const rootPeerId of rootPeerIds) { - const root = assignments.get(rootPeerId)!; - if (root.upstream.kind === "peer") { - const parent = assignments.get(root.upstream.peerId); - if (parent) { - assignments.set(root.upstream.peerId, { - ...parent, - childPeerIds: parent.childPeerIds.filter( - (childPeerId) => childPeerId !== rootPeerId, - ), - }); - } - } - assignments.set(rootPeerId, { - ...root, - upstream: { kind: "sfu" }, - childPeerIds: [], - }); - for (const childPeerId of root.childPeerIds) { - const child = assignments.get(childPeerId); - if ( - child?.upstream.kind === "peer" && - child.upstream.peerId === rootPeerId - ) { - assignments.set(childPeerId, { - ...child, - upstream: { kind: "none" }, - }); - } - } - } - const host = assignments.get(hostPeerId); - if (host && rootPeerIds.length > 0) { - const directChildLimit = Math.max(0, endpointMediaCopyCapacity - 1); - const directChildPeerIds = host.childPeerIds.slice(0, directChildLimit); - for (const suppressedPeerId of host.childPeerIds.slice(directChildLimit)) { - const suppressed = assignments.get(suppressedPeerId); - if (suppressed?.upstream.kind === "peer") { - assignments.set(suppressedPeerId, { - ...suppressed, - upstream: { kind: "none" }, - }); - } - } - assignments.set(hostPeerId, { - ...host, - childPeerIds: directChildPeerIds, - sfuPublicationGeneration: publicationGeneration, - }); - } - return assignments; +function preparedRouteCandidate( + operation: OperationSnapshot, + tuple: CandidateTuple, + connectionId: string, +): PreparedRouteCandidate { + return { + childPeerId: operation.childPeerId, + connectionId, + transport: tuple.kind === "peer" ? tuple.transport : "sfu", + }; } -function cloneAssignments( - assignments: ReadonlyMap, -): Map { - return new Map( - [...assignments].map(([peerId, assignment]) => [ - peerId, - { - upstream: - assignment.upstream.kind === "peer" - ? { kind: "peer" as const, peerId: assignment.upstream.peerId } - : { kind: assignment.upstream.kind }, - childPeerIds: [...assignment.childPeerIds], - sfuPublicationGeneration: assignment.sfuPublicationGeneration, - }, - ]), - ); +function opaqueId(): string { + return randomBytes(16).toString("base64url"); } -function promoteSfuRoots( - assignments: Map, - rootPeerIds: readonly string[], -): boolean { - for (const rootPeerId of rootPeerIds) { - const root = assignments.get(rootPeerId); - if (!root || root.childPeerIds.length > 0) { - return false; - } - if (root.upstream.kind === "peer") { - const parent = assignments.get(root.upstream.peerId); - if (!parent) { - return false; - } - assignments.set(root.upstream.peerId, { - ...parent, - childPeerIds: parent.childPeerIds.filter( - (childPeerId) => childPeerId !== rootPeerId, - ), - }); - } - assignments.set(rootPeerId, { - ...root, - upstream: { kind: "sfu" }, - }); - } - return true; +function overlapResource(endpointPeerId: string): OverlapRouteResource { + return { kind: "overlap", endpointPeerId, released: false }; } -function findHostBranchRoot( - assignments: ReadonlyMap, - peerId: string, - hostPeerId: string, -): string | undefined { - const visited = new Set(); - let branchPeerId = peerId; - while (!visited.has(branchPeerId)) { - visited.add(branchPeerId); - const assignment = assignments.get(branchPeerId); - if (assignment?.upstream.kind !== "peer") { - return undefined; - } - if (assignment.upstream.peerId === hostPeerId) { - return branchPeerId; - } - branchPeerId = assignment.upstream.peerId; - } - return undefined; +function turnResource(fence: SelectedEdgeTurnIdentity): TurnRouteResource { + return { kind: "turn", fence, released: false }; } -function sameUpstream( - left: ParticipantRouteAssignment, - right: ParticipantRouteAssignment, -): boolean { - return ( - left.upstream.kind === right.upstream.kind && - (left.upstream.kind !== "peer" || - (right.upstream.kind === "peer" && - left.upstream.peerId === right.upstream.peerId)) - ); +function subscriptionResource( + fence: SfuSubscriptionFence, +): SfuSubscriptionRouteResource { + return { kind: "sfu-subscription", fence, released: false }; } -function isHardBadViewerQualityWindow( - evidence: Extract, -): boolean { - const metrics = evidence.metrics; - const freezeRatio = - metrics.freezeDurationMsDelta === null - ? null - : metrics.freezeDurationMsDelta / evidence.windowMs; - if (freezeRatio !== null && freezeRatio >= VIEWER_QUALITY_FREEZE_RATIO) { - return true; - } - if ( - metrics.packetsReceivedDelta !== null && - metrics.packetsReceivedDelta > 0 && - metrics.framesDecodedDelta === 0 - ) { - return true; - } - if ( - metrics.packetsReceivedDelta === null || - metrics.packetsLostDelta === null - ) { - return false; - } - const totalPackets = - metrics.packetsReceivedDelta + metrics.packetsLostDelta; - return ( - totalPackets >= VIEWER_QUALITY_MIN_LOSS_PACKETS && - metrics.packetsLostDelta / totalPackets >= VIEWER_QUALITY_HIGH_LOSS_RATIO - ); +function publicationResource( + fence: SfuResourceFence, + ingressTurn?: TurnRouteResource, +): SfuPublicationRouteResource { + return { + kind: "sfu-publication", + fence, + released: false, + ...(ingressTurn ? { ingressTurn } : {}), + }; } -function isHardBadParentEdgeQualityProof( - proof: ParentEdgeQualityProof, -): boolean { - if (proof.kind === "sender-limited") { - return true; +function reservationResources( + reservation: CandidateReservation, +): RouteResource[] { + const resources: RouteResource[] = []; + if ("edge" in reservation) resources.push(reservation.edge); + if ("publication" in reservation) resources.push(reservation.publication); + if ("overlap" in reservation && reservation.overlap) { + resources.push(reservation.overlap); } - if (proof.kind !== "remote-loss") { - return false; - } - return ( - proof.packetsSentDelta >= VIEWER_QUALITY_MIN_LOSS_PACKETS && - proof.remotePacketsLostDelta / proof.packetsSentDelta >= - VIEWER_QUALITY_HIGH_LOSS_RATIO - ); + return resources; } diff --git a/src/server/media-route-controller.ts b/src/server/media-route-controller.ts deleted file mode 100644 index 31b6b4b..0000000 --- a/src/server/media-route-controller.ts +++ /dev/null @@ -1,461 +0,0 @@ -import { - MAX_MEDIA_ROUTE_REVISION, - MAX_VIEWERS_PER_ROOM_LIMIT, - participantRouteAssignmentSchema, - sfuPublicationGenerationSchema, - type MediaRoutePhase, - type ParticipantRouteAssignment, -} from "../shared/protocol.js"; -import { - assertEndpointMediaCopyCapacity, - assertEndpointMediaCopyCount, - countEndpointMediaCopies, - DEFAULT_ENDPOINT_MEDIA_COPY_CAPACITY, -} from "../shared/media-copy-accounting.js"; - -const MAX_PARTICIPANTS_PER_ROOM = MAX_VIEWERS_PER_ROOM_LIMIT + 1; - -export interface RoomSfuRoute { - publicationGeneration: string | null; - rootPeerIds: readonly string[]; -} - -export interface RoomMediaRoute { - revision: number; - assignments: ReadonlyMap; - sfu: RoomSfuRoute; -} - -export interface PendingRoomMediaRoute { - route: RoomMediaRoute; - expectedParticipantIds: ReadonlySet; - readyParticipantIds: ReadonlySet; -} - -export interface MediaRouteControllerOptions { - hostPeerId: string; - assignments: ReadonlyMap; - revision?: number; - sfuPublicationGeneration?: string | null; - sfuRootPeerIds?: readonly string[]; - endpointMediaCopyCapacity?: number; -} - -export interface PrepareMediaRouteInput { - assignments: ReadonlyMap; - expectedParticipantIds: ReadonlySet; - sfuPublicationGeneration?: string | null; - sfuRootPeerIds?: readonly string[]; -} - -export interface ReconcileBaselineInput { - assignments: ReadonlyMap; - sfuPublicationGeneration?: string | null; - sfuRootPeerIds?: readonly string[]; -} - -interface StoredPendingRoute { - route: RoomMediaRoute; - expectedParticipantIds: Set; - readyParticipantIds: Set; -} - -export class MediaRouteController { - private activeRoute: RoomMediaRoute; - private pendingRoute: StoredPendingRoute | undefined; - private latestRevision: number; - private readonly hostPeerId: string; - private readonly endpointMediaCopyCapacity: number; - - constructor(options: MediaRouteControllerOptions) { - this.hostPeerId = options.hostPeerId; - this.endpointMediaCopyCapacity = - options.endpointMediaCopyCapacity ?? - DEFAULT_ENDPOINT_MEDIA_COPY_CAPACITY; - assertEndpointMediaCopyCapacity(this.endpointMediaCopyCapacity); - const revision = options.revision ?? 0; - assertRevision(revision); - this.activeRoute = createRoute( - revision, - options.assignments, - options.sfuPublicationGeneration ?? null, - options.sfuRootPeerIds ?? [], - this.hostPeerId, - this.endpointMediaCopyCapacity, - ); - this.latestRevision = revision; - } - - getActiveRoute(): RoomMediaRoute { - return cloneRoute(this.activeRoute); - } - - getPendingRoute(): PendingRoomMediaRoute | undefined { - const pending = this.pendingRoute; - return pending - ? { - route: cloneRoute(pending.route), - expectedParticipantIds: new Set(pending.expectedParticipantIds), - readyParticipantIds: new Set(pending.readyParticipantIds), - } - : undefined; - } - - reconcileBaseline(input: ReconcileBaselineInput): number | undefined { - if (this.pendingRoute) { - return undefined; - } - - const revision = this.nextRevision(); - const route = createRoute( - revision, - input.assignments, - input.sfuPublicationGeneration === undefined - ? this.activeRoute.sfu.publicationGeneration - : input.sfuPublicationGeneration, - input.sfuRootPeerIds ?? this.activeRoute.sfu.rootPeerIds, - this.hostPeerId, - this.endpointMediaCopyCapacity, - ); - if (hasSameTopology(route, this.activeRoute)) { - return undefined; - } - - this.activeRoute = route; - this.latestRevision = revision; - return revision; - } - - prepare(input: PrepareMediaRouteInput): number | undefined { - if (this.pendingRoute) { - return undefined; - } - - const revision = this.nextRevision(); - const route = createRoute( - revision, - input.assignments, - input.sfuPublicationGeneration ?? null, - input.sfuRootPeerIds ?? [], - this.hostPeerId, - this.endpointMediaCopyCapacity, - ); - const expectedParticipantIds = new Set(input.expectedParticipantIds); - if (expectedParticipantIds.size > MAX_PARTICIPANTS_PER_ROOM) { - throw new Error("Too many expected media route participants"); - } - for (const peerId of expectedParticipantIds) { - if (!route.assignments.has(peerId)) { - throw new Error(`Expected participant ${peerId} has no route assignment`); - } - } - - this.latestRevision = revision; - this.pendingRoute = { - route, - expectedParticipantIds, - readyParticipantIds: new Set(), - }; - return revision; - } - - ready(peerId: string, revision: number, phase: MediaRoutePhase): boolean { - const pending = this.pendingRoute; - if ( - phase !== "prepare" || - !pending || - pending.route.revision !== revision || - !pending.expectedParticipantIds.has(peerId) || - pending.readyParticipantIds.has(peerId) - ) { - return false; - } - - pending.readyParticipantIds.add(peerId); - return true; - } - - commit(revision: number): boolean { - const pending = this.pendingRoute; - if (!pending || pending.route.revision !== revision) { - return false; - } - for (const peerId of pending.expectedParticipantIds) { - if (!pending.readyParticipantIds.has(peerId)) { - return false; - } - } - - this.activeRoute = pending.route; - this.pendingRoute = undefined; - return true; - } - - abort(revision: number): boolean { - const pending = this.pendingRoute; - if (!pending || pending.route.revision !== revision) { - return false; - } - - const rollbackRevision = this.nextRevision(); - this.activeRoute = cloneRoute(this.activeRoute, rollbackRevision); - this.latestRevision = rollbackRevision; - this.pendingRoute = undefined; - return true; - } - - hostActiveMediaEdges(): number { - const edgeCount = hostMediaEdges(this.activeRoute, this.hostPeerId); - assertEndpointMediaCopyCount( - edgeCount, - this.endpointMediaCopyCapacity, - ); - return edgeCount; - } - - private nextRevision(): number { - if (this.latestRevision >= MAX_MEDIA_ROUTE_REVISION) { - throw new Error("Media route revision space exhausted"); - } - return this.latestRevision + 1; - } -} - -function createRoute( - revision: number, - assignments: ReadonlyMap, - publicationGeneration: string | null, - rootPeerIds: readonly string[], - hostPeerId: string, - endpointMediaCopyCapacity: number, -): RoomMediaRoute { - assertRevision(revision); - if ( - assignments.size === 0 || - assignments.size > MAX_PARTICIPANTS_PER_ROOM - ) { - throw new Error("Media route participant count is outside room bounds"); - } - if (publicationGeneration !== null) { - sfuPublicationGenerationSchema.parse(publicationGeneration); - } - - const clonedAssignments = new Map(); - for (const [peerId, assignment] of assignments) { - const parsed = participantRouteAssignmentSchema.parse(assignment); - clonedAssignments.set(peerId, cloneAssignment(parsed)); - } - const route: RoomMediaRoute = { - revision, - assignments: clonedAssignments, - sfu: { - publicationGeneration, - rootPeerIds: [...rootPeerIds], - }, - }; - assertRouteInvariants( - route, - hostPeerId, - endpointMediaCopyCapacity, - ); - return route; -} - -function assertRouteInvariants( - route: RoomMediaRoute, - hostPeerId: string, - endpointMediaCopyCapacity: number, -): void { - const hostAssignment = route.assignments.get(hostPeerId); - if (!hostAssignment) { - throw new Error("Media route is missing its host assignment"); - } - if (hostAssignment.upstream.kind !== "none") { - throw new Error("The host cannot have a media upstream"); - } - if ( - hostAssignment.sfuPublicationGeneration !== - route.sfu.publicationGeneration - ) { - throw new Error("Host and room SFU publication generations differ"); - } - - const rootPeerIds = new Set(route.sfu.rootPeerIds); - if (rootPeerIds.size !== route.sfu.rootPeerIds.length) { - throw new Error("SFU root participants must be unique"); - } - if ( - (route.sfu.publicationGeneration === null) !== (rootPeerIds.size === 0) - ) { - throw new Error("SFU roots and publication generation must be active together"); - } - - for (const [peerId, assignment] of route.assignments) { - assertEndpointMediaCopyCount( - countEndpointMediaCopies({ - childPeerIds: assignment.childPeerIds, - publicationGeneration: assignment.sfuPublicationGeneration, - }), - endpointMediaCopyCapacity, - ); - if ( - peerId !== hostPeerId && - assignment.sfuPublicationGeneration !== null - ) { - throw new Error("Only the host may own an SFU publication generation"); - } - if (assignment.childPeerIds.includes(peerId)) { - throw new Error("A participant cannot be its own media child"); - } - - for (const childPeerId of assignment.childPeerIds) { - const childAssignment = route.assignments.get(childPeerId); - if (!childAssignment) { - throw new Error(`Media child ${childPeerId} has no assignment`); - } - if ( - childAssignment.upstream.kind !== "peer" || - childAssignment.upstream.peerId !== peerId - ) { - throw new Error("Peer media edges must be reciprocal"); - } - } - - if (assignment.upstream.kind === "peer") { - const parentAssignment = route.assignments.get( - assignment.upstream.peerId, - ); - if (!parentAssignment?.childPeerIds.includes(peerId)) { - throw new Error("Peer media edges must be reciprocal"); - } - } - if (assignment.upstream.kind === "sfu" && !rootPeerIds.has(peerId)) { - throw new Error("Every SFU upstream must belong to the root allowlist"); - } - if ( - assignment.upstream.kind === "sfu" && - assignment.childPeerIds.length > 0 - ) { - throw new Error("SFU-fed viewers require outbound relay proof"); - } - } - - for (const rootPeerId of rootPeerIds) { - const rootAssignment = route.assignments.get(rootPeerId); - if (!rootAssignment || rootAssignment.upstream.kind !== "sfu") { - throw new Error("Every SFU root must have the SFU as its upstream"); - } - } - - assertAcyclicPeerEdges(route.assignments); - assertEndpointMediaCopyCount( - hostMediaEdges(route, hostPeerId), - endpointMediaCopyCapacity, - ); -} - -function assertAcyclicPeerEdges( - assignments: ReadonlyMap, -): void { - for (const startPeerId of assignments.keys()) { - const ancestors = new Set([startPeerId]); - let assignment = assignments.get(startPeerId); - while (assignment?.upstream.kind === "peer") { - const parentPeerId = assignment.upstream.peerId; - if (ancestors.has(parentPeerId)) { - throw new Error("Peer media route contains a cycle"); - } - ancestors.add(parentPeerId); - assignment = assignments.get(parentPeerId); - } - } -} - -function hostMediaEdges(route: RoomMediaRoute, hostPeerId: string): number { - const hostAssignment = route.assignments.get(hostPeerId); - if (!hostAssignment) { - throw new Error("Media route is missing its host assignment"); - } - return countEndpointMediaCopies({ - childPeerIds: hostAssignment.childPeerIds, - publicationGeneration: route.sfu.publicationGeneration, - }); -} - -function hasSameTopology(left: RoomMediaRoute, right: RoomMediaRoute): boolean { - if ( - left.sfu.publicationGeneration !== right.sfu.publicationGeneration || - left.sfu.rootPeerIds.length !== right.sfu.rootPeerIds.length || - left.sfu.rootPeerIds.some( - (peerId, index) => peerId !== right.sfu.rootPeerIds[index], - ) || - left.assignments.size !== right.assignments.size - ) { - return false; - } - - for (const [peerId, leftAssignment] of left.assignments) { - const rightAssignment = right.assignments.get(peerId); - if ( - !rightAssignment || - leftAssignment.sfuPublicationGeneration !== - rightAssignment.sfuPublicationGeneration || - leftAssignment.upstream.kind !== rightAssignment.upstream.kind || - (leftAssignment.upstream.kind === "peer" && - (rightAssignment.upstream.kind !== "peer" || - leftAssignment.upstream.peerId !== - rightAssignment.upstream.peerId)) || - leftAssignment.childPeerIds.length !== - rightAssignment.childPeerIds.length || - leftAssignment.childPeerIds.some( - (childPeerId, index) => - childPeerId !== rightAssignment.childPeerIds[index], - ) - ) { - return false; - } - } - return true; -} - -function cloneRoute( - route: RoomMediaRoute, - revision = route.revision, -): RoomMediaRoute { - return { - revision, - assignments: new Map( - [...route.assignments].map(([peerId, assignment]) => [ - peerId, - cloneAssignment(assignment), - ]), - ), - sfu: { - publicationGeneration: route.sfu.publicationGeneration, - rootPeerIds: [...route.sfu.rootPeerIds], - }, - }; -} - -function cloneAssignment( - assignment: ParticipantRouteAssignment, -): ParticipantRouteAssignment { - return { - upstream: - assignment.upstream.kind === "peer" - ? { kind: "peer", peerId: assignment.upstream.peerId } - : { kind: assignment.upstream.kind }, - childPeerIds: [...assignment.childPeerIds], - sfuPublicationGeneration: assignment.sfuPublicationGeneration, - }; -} - -function assertRevision(revision: number): void { - if ( - !Number.isSafeInteger(revision) || - revision < 0 || - revision > MAX_MEDIA_ROUTE_REVISION - ) { - throw new Error("Media route revision is outside protocol bounds"); - } -} diff --git a/src/server/peer-relay-topology.ts b/src/server/peer-relay-topology.ts deleted file mode 100644 index 33b1c7d..0000000 --- a/src/server/peer-relay-topology.ts +++ /dev/null @@ -1,604 +0,0 @@ -import type { - MediaAssignment, - RelayDownstreamEdges, -} from "../shared/protocol.js"; -import { - assertEndpointMediaCopyCapacity, - DEFAULT_ENDPOINT_MEDIA_COPY_CAPACITY, -} from "../shared/media-copy-accounting.js"; - -export interface MediaAssignmentChange { - peerId: string; - previousParentPeerId?: string | null; - mediaAssignment: MediaAssignment; -} - -export interface RelayCapacityUpdateOptions { - rescueUnassignedRelay?: boolean; -} - -export const MAX_PEER_RELAY_DEPTH = 4; - -interface RelayViewer { - order: number; - parentPeerId: string | null; - childPeerIds: string[]; - downstreamEdges: RelayDownstreamEdges; -} - -interface RelayRoom { - hostPeerId?: string; - hostChildPeerIds: string[]; - endpointMediaCopyCapacity: RelayDownstreamEdges; - viewers: Map; - nextOrder: number; -} - -export class PeerRelayTopology { - private readonly rooms = new Map(); - private readonly endpointMediaCopyCapacity: RelayDownstreamEdges; - - constructor( - endpointMediaCopyCapacity: RelayDownstreamEdges = - DEFAULT_ENDPOINT_MEDIA_COPY_CAPACITY, - ) { - assertEndpointMediaCopyCapacity(endpointMediaCopyCapacity); - this.endpointMediaCopyCapacity = endpointMediaCopyCapacity; - } - - setHost( - roomId: string, - hostPeerId: string, - connectedPeerIds: ReadonlySet, - ): MediaAssignmentChange[] { - const room = this.room(roomId); - const before = snapshot(room); - const previousHostPeerId = room.hostPeerId; - room.hostPeerId = hostPeerId; - - if (previousHostPeerId !== hostPeerId) { - for (const childPeerId of room.hostChildPeerIds) { - const child = room.viewers.get(childPeerId); - if (child) { - child.parentPeerId = hostPeerId; - } - } - } - - for (const [peerId, viewer] of orderedViewers(room)) { - if (viewer.parentPeerId === null) { - this.assignViewer(room, peerId, connectedPeerIds); - } - } - return changedAssignments(before, snapshot(room)); - } - - addViewer( - roomId: string, - peerId: string, - connectedPeerIds: ReadonlySet, - ): MediaAssignmentChange[] { - const room = this.room(roomId); - const before = snapshot(room); - const existingViewer = room.viewers.get(peerId); - if (!existingViewer) { - room.viewers.set(peerId, { - order: room.nextOrder, - parentPeerId: null, - childPeerIds: [], - downstreamEdges: 0, - }); - room.nextOrder += 1; - } else { - existingViewer.downstreamEdges = 0; - } - for (const [candidatePeerId, viewer] of orderedViewers(room)) { - if (viewer.parentPeerId === null) { - this.assignViewer(room, candidatePeerId, connectedPeerIds); - } - } - return changedAssignments(before, snapshot(room)); - } - - setViewerRelayCapacity( - roomId: string, - peerId: string, - downstreamEdges: RelayDownstreamEdges, - connectedPeerIds: ReadonlySet, - options: RelayCapacityUpdateOptions = {}, - ): MediaAssignmentChange[] { - const room = this.rooms.get(roomId); - const viewer = room?.viewers.get(peerId); - if (!room || !viewer) { - return []; - } - - const before = snapshot(room); - const boundedDownstreamEdges = Math.min( - downstreamEdges, - room.endpointMediaCopyCapacity, - ); - const previousDownstreamEdges = viewer.downstreamEdges; - viewer.downstreamEdges = boundedDownstreamEdges; - for (const [candidatePeerId, candidate] of orderedViewers(room)) { - if (candidate.parentPeerId === null) { - this.assignViewer(room, candidatePeerId, connectedPeerIds); - } - } - if ( - options.rescueUnassignedRelay && - previousDownstreamEdges === 0 && - boundedDownstreamEdges > 0 - ) { - this.rescueUnassignedRelay(room, peerId, connectedPeerIds); - } - for (const [candidatePeerId, candidate] of orderedViewers(room)) { - if (candidate.parentPeerId === null) { - this.assignViewer(room, candidatePeerId, connectedPeerIds); - } - } - return changedAssignments(before, snapshot(room)); - } - - removeViewer( - roomId: string, - peerId: string, - connectedPeerIds: ReadonlySet, - ): MediaAssignmentChange[] { - const room = this.rooms.get(roomId); - const removed = room?.viewers.get(peerId); - if (!room || !removed) { - return []; - } - - const before = snapshot(room); - this.detach(room, peerId, removed.parentPeerId); - const subtreeRootPeerIds = [...removed.childPeerIds]; - for (const subtreeRootPeerId of subtreeRootPeerIds) { - const subtreeRoot = room.viewers.get(subtreeRootPeerId); - if (subtreeRoot) { - subtreeRoot.parentPeerId = null; - } - } - room.viewers.delete(peerId); - - for (const subtreeRootPeerId of subtreeRootPeerIds) { - this.assignViewer(room, subtreeRootPeerId, connectedPeerIds); - } - for (const [candidatePeerId, viewer] of orderedViewers(room)) { - if (viewer.parentPeerId === null) { - this.assignViewer(room, candidatePeerId, connectedPeerIds); - } - } - return changedAssignments(before, snapshot(room)); - } - - getAssignment( - roomId: string, - peerId: string, - ): MediaAssignment | undefined { - const room = this.rooms.get(roomId); - if (!room) { - return undefined; - } - if (room.hostPeerId === peerId) { - return { - parentPeerId: null, - childPeerIds: [...room.hostChildPeerIds], - }; - } - const viewer = room.viewers.get(peerId); - return viewer - ? { - parentPeerId: viewer.parentPeerId, - childPeerIds: [...viewer.childPeerIds], - } - : undefined; - } - - getAssignments(roomId: string): Map { - const room = this.rooms.get(roomId); - return room ? snapshot(room) : new Map(); - } - - getDownstreamCapacity( - roomId: string, - peerId: string, - ): RelayDownstreamEdges { - const room = this.rooms.get(roomId); - return room ? downstreamCapacity(room, peerId) : 0; - } - - getAdvertisedDownstreamCapacity( - roomId: string, - peerId: string, - ): RelayDownstreamEdges { - const room = this.rooms.get(roomId); - return room ? advertisedDownstreamCapacity(room, peerId) : 0; - } - - getHostPeerId(roomId: string): string | undefined { - return this.rooms.get(roomId)?.hostPeerId; - } - - reassignViewer( - roomId: string, - peerId: string, - connectedPeerIds: ReadonlySet, - excludedParentPeerIds: ReadonlySet, - maxDepth = MAX_PEER_RELAY_DEPTH, - requiredParentPeerId?: string, - ): MediaAssignmentChange[] | undefined { - const parentPeerId = this.findViewerReassignmentParent( - roomId, - peerId, - connectedPeerIds, - excludedParentPeerIds, - maxDepth, - ); - if (!parentPeerId || (requiredParentPeerId && parentPeerId !== requiredParentPeerId)) { - return undefined; - } - - const room = this.rooms.get(roomId); - const viewer = room?.viewers.get(peerId); - if (!room || !viewer) { - return undefined; - } - - const before = snapshot(room); - this.detach(room, peerId, viewer.parentPeerId); - viewer.parentPeerId = parentPeerId; - const children = childPeerIds(room, parentPeerId); - children.push(peerId); - children.sort( - (left, right) => - room.viewers.get(left)!.order - room.viewers.get(right)!.order, - ); - return changedAssignments(before, snapshot(room)); - } - - findViewerReassignmentParent( - roomId: string, - peerId: string, - connectedPeerIds: ReadonlySet, - excludedParentPeerIds: ReadonlySet, - maxDepth = MAX_PEER_RELAY_DEPTH, - ): string | undefined { - const room = this.rooms.get(roomId); - const viewer = room?.viewers.get(peerId); - if ( - !room?.hostPeerId || - !viewer || - !connectedPeerIds.has(peerId) || - !Number.isSafeInteger(maxDepth) || - maxDepth < 1 - ) { - return undefined; - } - - const subtreePeerIds = collectSubtreePeerIds(room, peerId); - const subtreeHeight = maximumSubtreeDepth(room, peerId); - if (maximumDepthOutsideSubtree(room, subtreePeerIds) > maxDepth) { - return undefined; - } - const parentPeerId = findReassignmentParent( - room, - connectedPeerIds, - excludedParentPeerIds, - subtreePeerIds, - maxDepth - subtreeHeight - 1, - ); - return parentPeerId === viewer.parentPeerId ? undefined : parentPeerId; - } - - isParentOf(roomId: string, parentPeerId: string, childPeerId: string): boolean { - return ( - this.rooms.get(roomId)?.viewers.get(childPeerId)?.parentPeerId === - parentPeerId - ); - } - - deleteRoom(roomId: string): void { - this.rooms.delete(roomId); - } - - private room(roomId: string): RelayRoom { - let room = this.rooms.get(roomId); - if (!room) { - room = { - hostChildPeerIds: [], - endpointMediaCopyCapacity: this.endpointMediaCopyCapacity, - viewers: new Map(), - nextOrder: 0, - }; - this.rooms.set(roomId, room); - } - return room; - } - - private assignViewer( - room: RelayRoom, - peerId: string, - connectedPeerIds: ReadonlySet, - ): void { - const viewer = room.viewers.get(peerId); - if ( - !viewer || - viewer.parentPeerId !== null || - !connectedPeerIds.has(peerId) - ) { - return; - } - const parentPeerId = findShallowestAvailableParent( - room, - connectedPeerIds, - MAX_PEER_RELAY_DEPTH - maximumSubtreeDepth(room, peerId) - 1, - ); - if (!parentPeerId) { - return; - } - viewer.parentPeerId = parentPeerId; - const children = childPeerIds(room, parentPeerId); - children.push(peerId); - children.sort( - (left, right) => - room.viewers.get(left)!.order - room.viewers.get(right)!.order, - ); - } - - private detach( - room: RelayRoom, - peerId: string, - parentPeerId: string | null, - ): void { - if (!parentPeerId) { - return; - } - const children = childPeerIds(room, parentPeerId); - const childIndex = children.indexOf(peerId); - if (childIndex !== -1) { - children.splice(childIndex, 1); - } - } - - private rescueUnassignedRelay( - room: RelayRoom, - peerId: string, - connectedPeerIds: ReadonlySet, - ): void { - const hostPeerId = room.hostPeerId; - const candidate = room.viewers.get(peerId); - if ( - !hostPeerId || - !candidate || - !connectedPeerIds.has(hostPeerId) || - !connectedPeerIds.has(peerId) || - candidate.parentPeerId !== null || - candidate.childPeerIds.length !== 0 || - downstreamCapacity(room, peerId) === 0 || - room.hostChildPeerIds.length !== downstreamCapacity(room, hostPeerId) - ) { - return; - } - - const leafPeerId = room.hostChildPeerIds.find((childPeerId) => { - const child = room.viewers.get(childPeerId); - return ( - connectedPeerIds.has(childPeerId) && - child?.parentPeerId === hostPeerId && - child.childPeerIds.length === 0 && - downstreamCapacity(room, childPeerId) === 0 - ); - }); - if (!leafPeerId) { - return; - } - - const leaf = room.viewers.get(leafPeerId)!; - const leafIndex = room.hostChildPeerIds.indexOf(leafPeerId); - room.hostChildPeerIds.splice(leafIndex, 1, peerId); - room.hostChildPeerIds.sort( - (left, right) => - room.viewers.get(left)!.order - room.viewers.get(right)!.order, - ); - candidate.parentPeerId = hostPeerId; - candidate.childPeerIds.push(leafPeerId); - leaf.parentPeerId = peerId; - } -} - -function findShallowestAvailableParent( - room: RelayRoom, - connectedPeerIds: ReadonlySet, - maximumParentDepth: number, -): string | undefined { - const hostPeerId = room.hostPeerId; - if (!hostPeerId || !connectedPeerIds.has(hostPeerId)) { - return undefined; - } - - const queue: Array<{ peerId: string; depth: number }> = [ - { peerId: hostPeerId, depth: 0 }, - ]; - while (queue.length > 0) { - const { peerId, depth } = queue.shift()!; - if (depth > maximumParentDepth || !connectedPeerIds.has(peerId)) { - continue; - } - const children = childPeerIds(room, peerId); - const capacity = downstreamCapacity(room, peerId); - if (children.length < capacity) { - return peerId; - } - for (const childPeerId of children) { - queue.push({ peerId: childPeerId, depth: depth + 1 }); - } - } - return undefined; -} - -function findReassignmentParent( - room: RelayRoom, - connectedPeerIds: ReadonlySet, - excludedParentPeerIds: ReadonlySet, - subtreePeerIds: ReadonlySet, - maximumParentDepth: number, -): string | undefined { - const hostPeerId = room.hostPeerId; - if (!hostPeerId || !connectedPeerIds.has(hostPeerId)) { - return undefined; - } - - const queue: Array<{ peerId: string; depth: number }> = [ - { peerId: hostPeerId, depth: 0 }, - ]; - while (queue.length > 0) { - const { peerId, depth } = queue.shift()!; - if (depth > maximumParentDepth || !connectedPeerIds.has(peerId)) { - continue; - } - const children = childPeerIds(room, peerId); - const capacity = downstreamCapacity(room, peerId); - if ( - children.length < capacity && - !excludedParentPeerIds.has(peerId) && - !subtreePeerIds.has(peerId) - ) { - return peerId; - } - for (const childPeerId of children) { - queue.push({ peerId: childPeerId, depth: depth + 1 }); - } - } - return undefined; -} - -function downstreamCapacity( - room: RelayRoom, - peerId: string, -): RelayDownstreamEdges { - return advertisedDownstreamCapacity(room, peerId); -} - -function advertisedDownstreamCapacity( - room: RelayRoom, - peerId: string, -): RelayDownstreamEdges { - return room.hostPeerId === peerId - ? room.endpointMediaCopyCapacity - : (room.viewers.get(peerId)?.downstreamEdges ?? 0); -} - -function collectSubtreePeerIds(room: RelayRoom, rootPeerId: string): Set { - const peerIds = new Set(); - const queue = [rootPeerId]; - while (queue.length > 0) { - const peerId = queue.shift()!; - if (peerIds.has(peerId)) { - continue; - } - peerIds.add(peerId); - queue.push(...(room.viewers.get(peerId)?.childPeerIds ?? [])); - } - return peerIds; -} - -function maximumSubtreeDepth(room: RelayRoom, rootPeerId: string): number { - let maximumDepth = 0; - const queue: Array<{ peerId: string; depth: number }> = [ - { peerId: rootPeerId, depth: 0 }, - ]; - while (queue.length > 0) { - const { peerId, depth } = queue.shift()!; - maximumDepth = Math.max(maximumDepth, depth); - for (const childPeerId of room.viewers.get(peerId)?.childPeerIds ?? []) { - queue.push({ peerId: childPeerId, depth: depth + 1 }); - } - } - return maximumDepth; -} - -function maximumDepthOutsideSubtree( - room: RelayRoom, - excludedPeerIds: ReadonlySet, -): number { - const hostPeerId = room.hostPeerId; - if (!hostPeerId) { - return 0; - } - let maximumDepth = 0; - const queue: Array<{ peerId: string; depth: number }> = [ - { peerId: hostPeerId, depth: 0 }, - ]; - while (queue.length > 0) { - const { peerId, depth } = queue.shift()!; - maximumDepth = Math.max(maximumDepth, depth); - for (const childPeerId of childPeerIds(room, peerId)) { - if (!excludedPeerIds.has(childPeerId)) { - queue.push({ peerId: childPeerId, depth: depth + 1 }); - } - } - } - return maximumDepth; -} - -function childPeerIds(room: RelayRoom, peerId: string): string[] { - if (room.hostPeerId === peerId) { - return room.hostChildPeerIds; - } - const viewer = room.viewers.get(peerId); - if (!viewer) { - throw new Error("Peer relay topology references an unknown parent"); - } - return viewer.childPeerIds; -} - -function orderedViewers(room: RelayRoom): Array<[string, RelayViewer]> { - return [...room.viewers.entries()].sort( - ([, left], [, right]) => left.order - right.order, - ); -} - -function snapshot(room: RelayRoom): Map { - const assignments = new Map(); - if (room.hostPeerId) { - assignments.set(room.hostPeerId, { - parentPeerId: null, - childPeerIds: [...room.hostChildPeerIds], - }); - } - for (const [peerId, viewer] of room.viewers) { - assignments.set(peerId, { - parentPeerId: viewer.parentPeerId, - childPeerIds: [...viewer.childPeerIds], - }); - } - return assignments; -} - -function changedAssignments( - before: ReadonlyMap, - after: ReadonlyMap, -): MediaAssignmentChange[] { - const changes: MediaAssignmentChange[] = []; - for (const [peerId, mediaAssignment] of after) { - const previous = before.get(peerId); - if ( - !previous || - previous.parentPeerId !== mediaAssignment.parentPeerId || - previous.childPeerIds.length !== mediaAssignment.childPeerIds.length || - previous.childPeerIds.some( - (childPeerId, index) => - childPeerId !== mediaAssignment.childPeerIds[index], - ) - ) { - changes.push({ - peerId, - previousParentPeerId: previous?.parentPeerId, - mediaAssignment, - }); - } - } - return changes; -} diff --git a/src/server/room-route-controller.ts b/src/server/room-route-controller.ts index 25f422f..7927334 100644 --- a/src/server/room-route-controller.ts +++ b/src/server/room-route-controller.ts @@ -96,6 +96,7 @@ interface Participant { joinOrder: number; effectiveDownstreamCapacity: number; blockedAtFactVersion?: number; + failedTuple?: { key: string; factVersion: number }; } interface Attempt { @@ -168,6 +169,7 @@ interface HostPublication { usable: boolean; physicalActive: boolean; resource: Resource; + failedIngress?: { ingress: "direct" | "selected-turn"; factVersion: number }; } export class RoomRouteController { @@ -176,6 +178,7 @@ export class RoomRouteController { private hostPublication: HostPublication | null = null; private operation?: ChildOperation; private revision = 0; + private latestRevision = 0; private factVersion = 0; private nextJoinOrder = 0; private paused = false; @@ -210,8 +213,14 @@ export class RoomRouteController { const released: Resource[] = []; const previousSessionId = current.sessionId; if (previousSessionId !== input.sessionId) { + const revisionBefore = this.revision; + released.push(...this.abortOperation()); current.sessionId = input.sessionId; - released.push(...this.rebindCommittedSession(input.peerId, input.sessionId)); + const rebound = this.rebindCommittedSession(input.peerId, input.sessionId); + released.push(...rebound.released); + if (rebound.retired && this.revision === revisionBefore) { + this.revision = this.allocateRevision(); + } changed = true; } if (current.departureConfirmed) { @@ -326,6 +335,10 @@ export class RoomRouteController { edge.usable = false; child.blockedAtFactVersion = undefined; this.touchFacts(); + child.failedTuple = { + key: edgeTupleKey(edge), + factVersion: this.factVersion, + }; } return true; } @@ -344,10 +357,42 @@ export class RoomRouteController { if (publication.usable) { publication.usable = false; this.touchFacts(); + publication.failedIngress = { + ingress: publication.ingress, + factVersion: this.factVersion, + }; } return true; } + adoptDirectConnection(input: EdgeGuard & { newConnectionId: string }): boolean { + const edge = this.upstreamByViewer.get(input.childPeerId); + if ( + this.operation?.childPeerId === input.childPeerId || + this.revision !== input.routeRevision || + !edge || + edge.kind !== "peer" || + edge.transport !== "direct" || + !edge.usable || + !edge.physicalActive || + edge.childSessionId !== input.childSessionId || + edge.parentSessionId !== input.parentSessionId || + edge.connectionId !== input.connectionId || + !input.newConnectionId + ) { + return false; + } + if (input.newConnectionId === edge.connectionId) return true; + edge.connectionId = input.newConnectionId; + const child = this.participants.get(input.childPeerId); + if (child) { + child.blockedAtFactVersion = undefined; + child.failedTuple = undefined; + } + this.touchFacts(); + return true; + } + retireCommittedTransport(guard: EdgeGuard): readonly Resource[] { const child = this.participants.get(guard.childPeerId); const edge = this.upstreamByViewer.get(guard.childPeerId); @@ -360,7 +405,7 @@ export class RoomRouteController { edge.physicalActive = false; edge.usable = false; child.blockedAtFactVersion = undefined; - this.revision = this.nextRevision(); + this.revision = this.allocateRevision(); if (this.operation) this.operation.baseRevision = this.revision; this.touchFacts(); return edge.transport === "direct" ? [] : [edge.resource]; @@ -380,7 +425,7 @@ export class RoomRouteController { publication.connectionId !== guard.connectionId) return []; publication.physicalActive = false; publication.usable = false; - this.revision = this.nextRevision(); + this.revision = this.allocateRevision(); if (this.operation) this.operation.baseRevision = this.revision; this.touchFacts(); return [publication.resource]; @@ -409,6 +454,10 @@ export class RoomRouteController { const child = this.participants.get(childPeerId)!; const candidates = this.buildCandidates(childPeerId, bootstrap !== undefined); if (candidates.length === 0) { + if (this.retireInvalidOperationEdge(childPeerId, released)) { + this.revision = this.allocateRevision(); + this.touchFacts(); + } child.blockedAtFactVersion = this.factVersion; continue; } @@ -470,7 +519,7 @@ export class RoomRouteController { } operation.current = { tuple, - revision: this.nextRevision(), + revision: this.allocateRevision(), connectionId: input.connectionId, childSessionId: operation.childSessionId, parentSessionId: tuple.kind === "peer" ? this.participants.get(tuple.parentPeerId)?.sessionId ?? undefined : undefined, @@ -512,6 +561,7 @@ export class RoomRouteController { } const released = [...validation.released]; const retirement = plan.endpointTransition.retire; + let restoreTuple: CandidateTuple | undefined; if (retirement.kind === "edge") { const edge = this.upstreamByViewer.get(retirement.childPeerId); if (!edge || edge.kind !== "peer" || !edge.physicalActive || @@ -522,6 +572,13 @@ export class RoomRouteController { edge.connectionId !== retirement.connectionId) { return { accepted: false, operation: this.operationSnapshot(), released }; } + if (edge.usable) { + restoreTuple = { + kind: "peer", + parentPeerId: edge.parentPeerId, + transport: edge.transport, + }; + } if (edge.transport === "selected-turn") released.push(edge.resource); this.upstreamByViewer.delete(retirement.childPeerId); } else { @@ -535,13 +592,13 @@ export class RoomRouteController { released.push(...this.removePublicationGeneration(retirement.generation)); } this.advanceActiveRevision(operation); + this.touchFacts(); const nextTuple = retirement.kind === "publication" && plan.tuple.kind === "sfu" && plan.tuple.publication === "replace" ? { ...plan.tuple, publication: "create" as const } : plan.tuple; - const nextPlan = this.planCandidate(operation.childPeerId, nextTuple); - if (!nextPlan) { - operation.cursor += 1; + this.replanRemaining(operation, nextTuple, restoreTuple); + if (operation.cursor >= operation.candidates.length) { const advanced = this.validateOrAdvance(nowMs); return { accepted: true, @@ -550,17 +607,29 @@ export class RoomRouteController { released: [...released, ...advanced.released], }; } - operation.candidates[operation.cursor] = nextPlan; return { accepted: true, operation: this.operationSnapshot(), released }; } - candidateReady(guard: CandidateGuard, nowMs: number): SettleResult { + candidateReady( + guard: CandidateGuard, + nowMs: number, + commitReservation: (reservation: CandidateReservation) => boolean = () => true, + ): SettleResult { const validation = this.validateOrAdvance(nowMs); const operation = this.operation; const attempt = operation?.current; if (!operation || !attempt || !this.guardMatches(guard, operation, attempt)) { return { accepted: false, exhausted: validation.exhausted, activeRevision: this.revision, released: validation.released }; } + if (!commitReservation(attempt.reservation)) { + const failed = this.validateOrAdvance(nowMs, guard); + return { + accepted: false, + exhausted: failed.exhausted, + activeRevision: this.revision, + released: [...validation.released, ...failed.released], + }; + } const released = [...validation.released, ...this.commitAttempt(operation, attempt)]; return { accepted: true, activeRevision: this.revision, released }; } @@ -578,6 +647,22 @@ export class RoomRouteController { return { accepted: validation.expired, exhausted: validation.exhausted, activeRevision: this.revision, released: validation.released }; } + dispose(): readonly Resource[] { + const resources = new Set(); + if (this.operation?.current) { + for (const resource of reservationResources(this.operation.current.reservation)) { + resources.add(resource); + } + } + for (const resource of this.committedResources()) resources.add(resource); + this.operation = undefined; + this.upstreamByViewer.clear(); + this.hostPublication = null; + this.participants.clear(); + this.paused = true; + return [...resources]; + } + private validateOrAdvance(nowMs: number, failedGuard?: CandidateGuard): { released: Resource[]; exhausted?: boolean; expired: boolean; consumedGuard: boolean; } { @@ -589,7 +674,7 @@ export class RoomRouteController { if (operation.current) released.push(...reservationResources(operation.current.reservation)); if (operation.current) this.advanceActiveRevision(operation); const factsChanged = operation.builtAtFactVersion !== this.factVersion; - this.blockAndClear(operation, !factsChanged); + this.blockAndClear(operation, !factsChanged, released); return { ...result, exhausted: !factsChanged, expired: true }; } if (this.participants.get(operation.childPeerId)?.sessionId !== operation.childSessionId) { @@ -601,6 +686,9 @@ export class RoomRouteController { while ((operation = this.operation)) { const attempt = operation.current; + if (!attempt && operation.builtAtFactVersion !== this.factVersion) { + this.replanRemaining(operation); + } if (attempt) { const guardFailed = failedGuard && this.guardMatches(failedGuard, operation, attempt); const plan = operation.candidates[operation.cursor]; @@ -610,6 +698,7 @@ export class RoomRouteController { operation.current = undefined; operation.cursor += 1; result.consumedGuard = Boolean(guardFailed); + this.replanRemaining(operation); } while (operation.cursor < operation.candidates.length && !this.candidateValid(operation.childPeerId, operation.candidates[operation.cursor]!)) { @@ -617,21 +706,18 @@ export class RoomRouteController { } if (operation.cursor < operation.candidates.length) return result; const factsChanged = operation.builtAtFactVersion !== this.factVersion; - this.blockAndClear(operation, !factsChanged); + this.blockAndClear(operation, !factsChanged, released); result.exhausted = !factsChanged; } return result; } private commitAttempt(operation: ChildOperation, attempt: Attempt): Resource[] { - const released: Resource[] = []; + const beforeResources = new Set(this.committedResources()); const old = this.upstreamByViewer.get(operation.childPeerId); - if ("overlap" in attempt.reservation && attempt.reservation.overlap !== undefined) released.push(attempt.reservation.overlap); if (attempt.tuple.kind === "sfu" && attempt.tuple.publication === "replace") { - released.push(...this.removePublicationGeneration(this.hostPublication!.generation)); - } else if (old && old.transport !== "direct" && old.physicalActive) { - released.push(old.resource); + this.removePublicationGeneration(this.hostPublication!.generation); } if (attempt.tuple.kind === "peer") { @@ -676,18 +762,30 @@ export class RoomRouteController { this.operation = undefined; this.touchFacts(); const participant = this.participants.get(operation.childPeerId); - if (participant) participant.blockedAtFactVersion = undefined; + if (participant) { + participant.blockedAtFactVersion = undefined; + participant.failedTuple = undefined; + } if (old?.kind === "sfu" && !this.hasSfuSubscribers() && this.hostPublication) { - if (this.hostPublication.physicalActive) released.push(this.hostPublication.resource); this.hostPublication = null; } this.assertGraph(); + const afterResources = new Set(this.committedResources()); + const released = [...beforeResources].filter((resource) => !afterResources.has(resource)); + if ("overlap" in attempt.reservation && attempt.reservation.overlap !== undefined) { + released.push(attempt.reservation.overlap); + } return released; } private buildCandidates(childPeerId: string, sfuOnly: boolean): CandidatePlan[] { - const failed = this.upstreamByViewer.get(childPeerId); - const failedKey = failed?.usable === false ? edgeTupleKey(failed) : undefined; + const child = this.participants.get(childPeerId)!; + const failedKey = child.failedTuple?.factVersion === this.factVersion + ? child.failedTuple.key + : undefined; + const failedIngress = this.hostPublication?.failedIngress?.factVersion === this.factVersion + ? this.hostPublication.failedIngress.ingress + : undefined; const descendants = this.descendantsOf(childPeerId); const parents = [...this.participants.values()] .filter((parent) => parent.sessionId && !parent.departureConfirmed && parent.peerId !== childPeerId && @@ -716,6 +814,7 @@ export class RoomRouteController { } } return candidates.filter((candidate, index, all) => tupleKey(candidate) !== failedKey && + !(candidate.kind === "sfu" && candidate.publication !== "reuse" && candidate.ingress === failedIngress) && all.findIndex((other) => tupleKey(other) === tupleKey(candidate)) === index) .map((candidate) => this.planCandidate(childPeerId, candidate)) .filter((candidate): candidate is CandidatePlan => candidate !== null && @@ -749,7 +848,7 @@ export class RoomRouteController { const participant = this.participants.get(id); const edge = this.upstreamByViewer.get(id); return participant?.sessionId && participant.blockedAtFactVersion !== this.factVersion && - edge?.kind === "peer" && edge.transport === "direct" && edge.usable; + edge?.kind === "peer" && edge.transport === "direct" && edge.usable && edge.physicalActive; }) .sort((left, right) => compareParticipant(this.participants.get(right)!, this.participants.get(left)!))[0]; } @@ -764,16 +863,29 @@ export class RoomRouteController { private overflowChildren(): string[] { const result: string[] = []; for (const parent of this.participants.values()) { - const capacity = Math.max(0, parent.effectiveDownstreamCapacity - (parent.role === "host" && this.hostPublication ? 1 : 0)); - const children = this.childrenOf(parent.peerId).filter((id) => { + const overflow = this.overflowPeerChildren(parent.peerId).filter((id) => { const child = this.participants.get(id); return child?.sessionId && !child.departureConfirmed && child.blockedAtFactVersion !== this.factVersion; - }).sort((left, right) => compareParticipant(this.participants.get(left)!, this.participants.get(right)!)); - result.push(...children.slice(capacity).reverse()); + }); + result.push(...overflow.reverse()); } return result; } + private overflowPeerChildren(parentPeerId: string): string[] { + const parent = this.participants.get(parentPeerId); + if (!parent) return []; + const publicationCopies = parent.role === "host" && this.hostPublication?.physicalActive ? 1 : 0; + const capacity = Math.max(0, parent.effectiveDownstreamCapacity - publicationCopies); + return this.childrenOf(parentPeerId) + .filter((id) => { + const edge = this.upstreamByViewer.get(id); + return edge?.kind === "peer" && edge.physicalActive; + }) + .sort((left, right) => compareParticipant(this.participants.get(left)!, this.participants.get(right)!)) + .slice(capacity); + } + private candidateValid(childPeerId: string, plan: CandidatePlan, attempt?: Attempt): boolean { const tuple = plan.tuple; const child = this.participants.get(childPeerId); @@ -836,7 +948,7 @@ export class RoomRouteController { } if (producerPeerId !== this.options.hostPeerId || !this.hostPublication?.physicalActive) return null; const replacesPublication = tuple.kind === "sfu" && tuple.publication === "replace"; - const releasesLastPublication = tuple.kind === "peer" && old?.kind === "sfu" && + const releasesLastPublication = tuple.kind === "peer" && old?.kind === "sfu" && old.physicalActive && this.sfuSubscriberCount() === 1; if (!replacesPublication && !releasesLastPublication) return null; return { @@ -883,7 +995,7 @@ export class RoomRouteController { if (this.hostPublication.physicalActive) released.push(this.hostPublication.resource); this.hostPublication = null; } - this.revision = this.nextRevision(); + this.revision = this.allocateRevision(); this.touchFacts(); } return removed; @@ -898,14 +1010,71 @@ export class RoomRouteController { } private advanceActiveRevision(operation: ChildOperation): void { - this.revision = this.nextRevision(); + this.revision = this.allocateRevision(); operation.baseRevision = this.revision; } - private blockAndClear(operation: ChildOperation, block = true): void { + private blockAndClear( + operation: ChildOperation, + block = true, + released: Resource[] = [], + ): void { const child = this.participants.get(operation.childPeerId); - if (block && child) child.blockedAtFactVersion = this.factVersion; this.operation = undefined; + if (!block || !child) return; + if (this.retireInvalidOperationEdge(operation.childPeerId, released)) { + this.revision = this.allocateRevision(); + this.touchFacts(); + } + child.blockedAtFactVersion = this.factVersion; + } + + private retireInvalidOperationEdge(childPeerId: string, released: Resource[]): boolean { + const edge = this.upstreamByViewer.get(childPeerId); + if (!edge || !this.edgeRequiresMove(childPeerId, edge)) return false; + if (edge.transport !== "direct" && edge.physicalActive) released.push(edge.resource); + edge.physicalActive = false; + edge.usable = false; + if (edge.kind === "sfu" && !this.hasSfuSubscribers() && this.hostPublication) { + if (this.hostPublication.physicalActive) released.push(this.hostPublication.resource); + this.hostPublication.physicalActive = false; + this.hostPublication.usable = false; + } + return true; + } + + private edgeRequiresMove(childPeerId: string, edge: CommittedEdge): boolean { + if (!edge.usable || !edge.physicalActive) return true; + if (edge.kind === "sfu") { + return !this.hostPublication?.usable || + !this.hostPublication.physicalActive || + this.hostPublication.generation !== edge.publicationGeneration; + } + if (this.participants.get(edge.parentPeerId)?.departureConfirmed) return true; + return this.overflowPeerChildren(edge.parentPeerId).includes(childPeerId); + } + + private replanRemaining( + operation: ChildOperation, + firstTuple?: CandidateTuple, + restoreTuple?: CandidateTuple, + ): void { + const prefix = operation.candidates.slice(0, operation.cursor); + const tuples = operation.candidates + .slice(operation.cursor) + .map((candidate) => candidate.tuple); + if (firstTuple) { + if (tuples.length === 0) tuples.push(firstTuple); + else tuples[0] = firstTuple; + } + if (restoreTuple && !tuples.some((tuple) => tupleKey(tuple) === tupleKey(restoreTuple))) { + tuples.push(restoreTuple); + } + const replanned = tuples + .map((tuple) => this.planCandidate(operation.childPeerId, tuple)) + .filter((plan): plan is CandidatePlan => plan !== null); + operation.candidates = [...prefix, ...replanned]; + operation.builtAtFactVersion = this.factVersion; } private operationSnapshot(): OperationSnapshot | undefined { @@ -963,8 +1132,8 @@ export class RoomRouteController { private targetFits(parentPeerId: string, childPeerId: string): boolean { const parent = this.participants.get(parentPeerId)!; const edge = this.upstreamByViewer.get(childPeerId); - const already = edge?.kind === "peer" && edge.parentPeerId === parentPeerId; - const releasesLastPublication = parent.role === "host" && edge?.kind === "sfu" && + const already = edge?.kind === "peer" && edge.parentPeerId === parentPeerId && edge.physicalActive; + const releasesLastPublication = parent.role === "host" && edge?.kind === "sfu" && edge.physicalActive && this.sfuSubscriberCount() === 1; return this.usedSlots(parentPeerId) - (releasesLastPublication ? 1 : 0) + (already ? 0 : 1) <= parent.effectiveDownstreamCapacity; @@ -973,9 +1142,10 @@ export class RoomRouteController { private publicationFits(childPeerId: string, replacing: boolean): boolean { const host = this.participants.get(this.options.hostPeerId); const edge = this.upstreamByViewer.get(childPeerId); - const releasesHost = edge?.kind === "peer" && edge.parentPeerId === this.options.hostPeerId; + const releasesHost = edge?.kind === "peer" && edge.parentPeerId === this.options.hostPeerId && edge.physicalActive; + const replacesPhysicalPublication = replacing && Boolean(this.hostPublication?.physicalActive); const nextSlots = this.usedSlots(this.options.hostPeerId) - (releasesHost ? 1 : 0) + - (replacing ? 0 : 1); + (replacesPhysicalPublication ? 0 : 1); return Boolean(host?.sessionId && !host.departureConfirmed && nextSlots <= host.effectiveDownstreamCapacity); } @@ -985,14 +1155,13 @@ export class RoomRouteController { } private usedSlots(peerId: string): number { - const participant = this.participants.get(peerId); - return this.childrenOf(peerId).length + (participant?.role === "host" && this.hostPublication ? 1 : 0); + return this.physicalCopies(peerId); } private remaining(parentPeerId: string, childPeerId: string): number { const parent = this.participants.get(parentPeerId)!; const edge = this.upstreamByViewer.get(childPeerId); - const already = edge?.kind === "peer" && edge.parentPeerId === parentPeerId; + const already = edge?.kind === "peer" && edge.parentPeerId === parentPeerId && edge.physicalActive; return parent.effectiveDownstreamCapacity - this.usedSlots(parentPeerId) - (already ? 0 : 1); } @@ -1012,43 +1181,59 @@ export class RoomRouteController { } private hasSfuSubscribers(): boolean { - return [...this.upstreamByViewer.values()].some((edge) => edge.kind === "sfu"); + return [...this.upstreamByViewer.values()].some((edge) => edge.kind === "sfu" && edge.physicalActive); } private sfuSubscriberCount(): number { - return [...this.upstreamByViewer.values()].filter((edge) => edge.kind === "sfu").length; + return [...this.upstreamByViewer.values()].filter((edge) => edge.kind === "sfu" && edge.physicalActive).length; } - private rebindCommittedSession(peerId: string, sessionId: string): Resource[] { - const released: Resource[] = []; + private committedResources(): Resource[] { + const resources: Resource[] = []; + for (const edge of this.upstreamByViewer.values()) { + if (edge.transport !== "direct" && edge.physicalActive) resources.push(edge.resource); + } + if (this.hostPublication?.physicalActive) resources.push(this.hostPublication.resource); + return resources; + } + + private rebindCommittedSession( + peerId: string, + sessionId: string, + ): { released: Resource[]; retired: boolean } { + const released = new Set(); + let retired = false; const ownEdge = this.upstreamByViewer.get(peerId); if (ownEdge?.kind === "peer" && ownEdge.transport === "selected-turn") { - if (ownEdge.physicalActive) released.push(ownEdge.resource); + if (ownEdge.physicalActive) released.add(ownEdge.resource); ownEdge.physicalActive = false; ownEdge.usable = false; + retired = true; } else if (ownEdge) { ownEdge.childSessionId = sessionId; } for (const edge of this.upstreamByViewer.values()) { if (edge.kind !== "peer" || edge.parentPeerId !== peerId) continue; if (edge.transport === "selected-turn") { - if (edge.physicalActive) released.push(edge.resource); + if (edge.physicalActive) released.add(edge.resource); edge.physicalActive = false; edge.usable = false; + retired = true; } else { edge.parentSessionId = sessionId; } } if (peerId === this.options.hostPeerId && this.hostPublication) { if (this.hostPublication.ingress === "selected-turn") { - if (this.hostPublication.physicalActive) released.push(this.hostPublication.resource); + if (this.hostPublication.physicalActive) released.add(this.hostPublication.resource); this.hostPublication.physicalActive = false; this.hostPublication.usable = false; + retired = true; } else { this.hostPublication.hostSessionId = sessionId; } } - return released; + return { released: [...released], retired }; } private removePublicationGeneration(generation: string): Resource[] { @@ -1099,9 +1284,10 @@ export class RoomRouteController { private touchFacts(): void { this.factVersion += 1; } - private nextRevision(): number { - if (this.revision >= MAX_MEDIA_ROUTE_REVISION) throw new Error("Media route revision space exhausted"); - return this.revision + 1; + private allocateRevision(): number { + if (this.latestRevision >= MAX_MEDIA_ROUTE_REVISION) throw new Error("Media route revision space exhausted"); + this.latestRevision += 1; + return this.latestRevision; } } diff --git a/src/server/sfu-resource-admission.ts b/src/server/sfu-resource-admission.ts index 5349d68..342c196 100644 --- a/src/server/sfu-resource-admission.ts +++ b/src/server/sfu-resource-admission.ts @@ -30,7 +30,6 @@ interface PublicationEntry { fence: SfuResourceFence; state: ResourceState; subscriptions: Map; - legacyEgress?: number; } interface RoomResources { @@ -115,6 +114,9 @@ export class SfuResourceAdmission { } if (publication.state === "committed") return []; if (publication.state !== "reserved") return null; + if (![...publication.subscriptions.values()].some((entry) => entry.state === "reserved")) { + return null; + } const draining: SfuResourceFence[] = []; for (const other of room.publications.values()) { @@ -177,54 +179,6 @@ export class SfuResourceAdmission { return true; } - /** Compatibility wrapper for the pre-controller aggregate reservation. */ - reserve(fence: SfuResourceFence, egress: number): boolean { - assertFence(fence); - assertPositiveSafeInteger(egress, "SFU egress reservation"); - const existing = this.publication(fence); - if (existing) { - return ( - existing.state !== "draining" && existing.legacyEgress === egress - ); - } - const room = this.rooms.get(fence.roomId); - if ( - this.ingressInUse >= this.ingressCapacity || - egress > this.egressCapacity - this.egressInUse || - [...(room?.publications.values() ?? [])].some( - (publication) => publication.state === "reserved", - ) - ) { - return false; - } - - const resources = room ?? { publications: new Map() }; - const publication: PublicationEntry = { - fence: cloneFence(fence), - state: "reserved", - subscriptions: new Map(), - legacyEgress: egress, - }; - for (let index = 0; index < egress; index += 1) { - const viewerPeerId = `legacy_${index}`; - publication.subscriptions.set(viewerPeerId, { - fence: { ...cloneFence(fence), viewerPeerId }, - state: "reserved", - reservedFromDraining: false, - }); - } - resources.publications.set(publicationKey(fence), publication); - this.rooms.set(fence.roomId, resources); - this.ingressInUse += 1; - this.egressInUse += egress; - return true; - } - - /** Compatibility wrapper for the pre-controller aggregate commit. */ - commit(fence: SfuResourceFence): readonly SfuResourceFence[] | null { - return this.commitPublication(fence); - } - beginDrain(fence: SfuResourceFence): boolean { assertFence(fence); const publication = this.publication(fence); diff --git a/src/server/signaling.ts b/src/server/signaling.ts index cdfd6ff..198928e 100644 --- a/src/server/signaling.ts +++ b/src/server/signaling.ts @@ -549,7 +549,7 @@ export class SignalingServer { ? this.hybridMediaRouter!.connectParticipant(routeParticipant) : undefined; } catch { - console.error("Peer relay topology did not assign an authenticated participant"); + console.error("Media router did not assign an authenticated participant"); this.sendError(socket, "SERVER_ERROR", "Media assignment failed"); socket.close(1011, "Media assignment failed"); return; @@ -629,6 +629,13 @@ export class SignalingServer { } if (hybridState) { + if (participant.role === "host") { + this.hybridMediaRouter!.setPaused( + participant.roomId, + this.pausedShareGenerationsByRoom.get(participant.roomId) === + shareGeneration, + ); + } this.hybridMediaRouter!.completeAuthentication( routeParticipant, hybridState, @@ -785,16 +792,6 @@ export class SignalingServer { message.downstreamEdges, ); return; - case "sfu-reselection-ready": - if (!this.isHybridMediaEnabled()) { - this.sendError(socket, "FORBIDDEN", "Media routes are not enabled"); - return; - } - this.hybridMediaRouter!.handleHealthySfuReselection( - { ...authenticated, sessionId: this.socketStates.get(socket)!.sessionId }, - message.revision, - ); - return; case "route-ready": if (!this.isHybridMediaEnabled()) { this.sendError(socket, "FORBIDDEN", "Media routes are not enabled"); @@ -940,6 +937,9 @@ export class SignalingServer { } else { this.pausedShareGenerationsByRoom.delete(authenticated.roomId); } + if (this.isHybridMediaEnabled()) { + this.hybridMediaRouter!.setPaused(authenticated.roomId, message.paused); + } for (const viewer of this.options.roomStore.getConnectedViewers( authenticated.roomId, )) { @@ -1205,15 +1205,6 @@ export class SignalingServer { acceptedAtMs: now, parentEvidenceAccepted: false, }); - if (this.isHybridMediaEnabled()) { - this.hybridMediaRouter!.handleViewerQualityEvidence({ - roomId: source.roomId, - viewerSessionId: viewerState.sessionId, - parentSessionId: parent.sessionId, - evidence: forwarded, - }); - } - const host = this.options.roomStore.getConnectedHost(source.roomId); if (!host || host.sessionId === parent.sessionId) { return; @@ -1282,12 +1273,6 @@ export class SignalingServer { } gate.parentEvidenceAccepted = true; - this.hybridMediaRouter!.handleParentEdgeQualityEvidence({ - roomId: source.roomId, - viewerSessionId: viewer.sessionId, - parentSessionId: sourceState.sessionId, - evidence: message, - }); } private routeSignal( @@ -1517,6 +1502,7 @@ export class SignalingServer { this.hybridMediaRouter!.disconnectParticipant( disconnected.roomId, disconnected.peerId, + state.sessionId, ); } if (disconnected.role === "host") { diff --git a/src/shared/protocol.ts b/src/shared/protocol.ts index fd21e93..3e1f795 100644 --- a/src/shared/protocol.ts +++ b/src/shared/protocol.ts @@ -6,7 +6,7 @@ import { isCanonicalVideoCodecEvidence } from "./video-codec-evidence.js"; export const MAX_VIEWERS_PER_ROOM_LIMIT = 20; export const MAX_PARTICIPANTS_PER_ROOM_LIMIT = MAX_VIEWERS_PER_ROOM_LIMIT + 1; export const MAX_SIGNAL_BYTES = 64 * 1024; -export const SIGNALING_PROTOCOL = "screener-v6"; +export const SIGNALING_PROTOCOL = "screener-v7"; export const ROOM_CODE_LENGTH = 12; export const MAX_MEDIA_ROUTE_REVISION = Number.MAX_SAFE_INTEGER; export const MAX_SFU_TOKEN_LENGTH = 8 * 1024; @@ -343,6 +343,17 @@ export const signalPayloadSchema = z.discriminatedUnion("kind", [ ]); export type SignalPayload = z.infer; +export const preparedRouteCandidateSchema = z + .object({ + childPeerId: opaqueIdSchema, + connectionId: opaqueIdSchema, + transport: z.enum(["direct", "selected-turn", "sfu"]), + }) + .strict(); +export type PreparedRouteCandidate = z.infer< + typeof preparedRouteCandidateSchema +>; + export const mediaAssignmentSchema = z .object({ parentPeerId: opaqueIdSchema.nullable(), @@ -571,12 +582,6 @@ export const clientMessageSchema = z.union([ downstreamEdges: relayDownstreamEdgesSchema, }) .strict(), - z - .object({ - type: z.literal("sfu-reselection-ready"), - revision: mediaRouteRevisionSchema, - }) - .strict(), z .object({ type: z.literal("route-ready"), @@ -736,14 +741,25 @@ export const serverMessageSchema = z.union([ mediaAssignment: mediaAssignmentSchema, }) .strict(), - z - .object({ - type: z.literal("route-update"), - revision: mediaRouteRevisionSchema, - phase: mediaRoutePhaseSchema, - assignment: participantRouteAssignmentSchema, - }) - .strict(), + z.discriminatedUnion("phase", [ + z + .object({ + type: z.literal("route-update"), + revision: mediaRouteRevisionSchema, + phase: z.literal("prepare"), + assignment: participantRouteAssignmentSchema, + candidate: preparedRouteCandidateSchema, + }) + .strict(), + z + .object({ + type: z.literal("route-update"), + revision: mediaRouteRevisionSchema, + phase: z.literal("active"), + assignment: participantRouteAssignmentSchema, + }) + .strict(), + ]), z .object({ type: z.literal("sfu-config"), diff --git a/tests/client-state.test.ts b/tests/client-state.test.ts index 553ecc8..0e1564f 100644 --- a/tests/client-state.test.ts +++ b/tests/client-state.test.ts @@ -640,7 +640,7 @@ describe("client signaling recovery policy", () => { Object.defineProperty(authenticated, "data", { value: JSON.stringify({ type: "authenticated", - protocol: "screener-v6", + protocol: "screener-v7", role: "viewer", peerId: "viewer_12345678", roomExpiresAt: null, @@ -714,7 +714,7 @@ describe("client signaling recovery policy", () => { socket.dispatchEvent(new Event("open")); receive(socket, { type: "authenticated", - protocol: "screener-v6", + protocol: "screener-v7", role: "host", peerId: "host_12345678", roomExpiresAt: null, @@ -839,7 +839,7 @@ describe("client signaling recovery policy", () => { sockets[0]!.dispatchEvent(new Event("open")); receive({ type: "authenticated", - protocol: "screener-v6", + protocol: "screener-v7", role: "viewer", peerId: "viewer_12345678", roomExpiresAt: null, @@ -967,7 +967,7 @@ describe("client signaling recovery policy", () => { JSON.parse(String(sockets[0]!.send.mock.calls[0]![0])), ).toMatchObject({ type: "authenticate", - protocol: "screener-v6", + protocol: "screener-v7", }); const message = new Event("message"); Object.defineProperty(message, "data", { value: payload }); diff --git a/tests/decoded-frame-stall.test.ts b/tests/decoded-frame-stall.test.ts new file mode 100644 index 0000000..05c1d8e --- /dev/null +++ b/tests/decoded-frame-stall.test.ts @@ -0,0 +1,44 @@ +import { describe, expect, it } from "vitest"; + +import { + DecodedFrameStallDetector, + ROUTE_DECODED_FRAME_STALL_MS, +} from "../src/client/media/decoded-frame-stall.ts"; + +describe("DecodedFrameStallDetector", () => { + it("reports one exact active stall and resets on progress or identity change", () => { + const detector = new DecodedFrameStallDetector(); + expect(detector.observe("peer:1:a", 0, 0)).toBe(false); + expect( + detector.observe("peer:1:a", 0, ROUTE_DECODED_FRAME_STALL_MS), + ).toBe(true); + expect( + detector.observe("peer:1:a", 0, ROUTE_DECODED_FRAME_STALL_MS + 1), + ).toBe(false); + expect(detector.observe("peer:1:a", 1, ROUTE_DECODED_FRAME_STALL_MS + 2)).toBe( + false, + ); + expect( + detector.observe("peer:1:a", 0, ROUTE_DECODED_FRAME_STALL_MS * 2 + 2), + ).toBe(true); + expect(detector.observe("sfu:2:b", 0, ROUTE_DECODED_FRAME_STALL_MS * 3)).toBe( + false, + ); + }); + + it("suppresses and rebases progress while paused", () => { + const detector = new DecodedFrameStallDetector(); + detector.observe("peer:1:a", 0, 0); + detector.setPaused(true, ROUTE_DECODED_FRAME_STALL_MS); + expect( + detector.observe("peer:1:a", 0, ROUTE_DECODED_FRAME_STALL_MS * 3), + ).toBe(false); + detector.setPaused(false, ROUTE_DECODED_FRAME_STALL_MS * 3); + expect( + detector.observe("peer:1:a", 0, ROUTE_DECODED_FRAME_STALL_MS * 4 - 1), + ).toBe(false); + expect( + detector.observe("peer:1:a", 0, ROUTE_DECODED_FRAME_STALL_MS * 4), + ).toBe(true); + }); +}); diff --git a/tests/host-peer-tracks.test.ts b/tests/host-peer-tracks.test.ts index 1f2f79a..01aca65 100644 --- a/tests/host-peer-tracks.test.ts +++ b/tests/host-peer-tracks.test.ts @@ -1009,6 +1009,11 @@ function hostProvisionalInput( ) { return { revision, + candidate: { + childPeerId: childPeerIds.at(-1) ?? "candidate-child", + connectionId: `candidate-connection-${revision}`, + transport: "direct" as const, + }, assignment: hostAssignment( childPeerIds, overrides.publicationGeneration ?? null, @@ -1138,49 +1143,16 @@ describe("Host provisional child runtime ownership", () => { } }); - it("replaces an active child with selected TURN only on active promotion", async () => { - const signals: Array<{ peerId: string; connectionId: string }> = []; - const activeConnection = { id: "active-connection" }; - const owner = new HostProvisionalChild({ - sendSignal: (peerId, payload) => { - signals.push({ peerId, connectionId: payload.connectionId }); - return true; - }, - activeConnectionId: () => activeConnection.id, - }); - const input = hostProvisionalInput( - 14, - ["selected-child"], - createStream(createTrack("video", "selected-video"), null), - { activeChildPeerIds: ["selected-child"] }, - ); - expect(owner.prepare(input)).toBe(true); - expect(FakePeerConnection.latest).toBeNull(); - expect(owner.prepareSelectedTurn({ - ...selectedEdgeTurn( - "selected-child", - activeConnection.id, - "selected-connection", - ), - revision: 14, - }, input)).toBe(true); - await vi.waitFor(() => expect(signals).toHaveLength(1)); - expect(signals[0]?.connectionId).toBe("selected-connection"); - - const activation = owner.activate({ - revision: 14, - assignment: input.assignment, - activeChildPeerIds: ["selected-child"], - maxMediaEdges: 2, - }); - expect(activation.kind).toBe("promote"); - if (activation.kind === "promote") { - activation.peer.dispose(); - } - }); }); describe("ViewerRelay downstream ownership", () => { + const routeCandidate = ( + revision: number, + childPeerId: string, + transport: "direct" | "selected-turn" = "direct", + connectionId = `relay-candidate-${revision}`, + ) => ({ childPeerId, connectionId, transport }); + it("promotes the exact prepared child connection within the current Viewer cap", async () => { const signals: Array<{ peerId: string; connectionId: string }> = []; const relay = new ViewerRelay( @@ -1194,7 +1166,7 @@ describe("ViewerRelay downstream ownership", () => { }, ); relay.setStream(createStream(createTrack("video", "prepared-video"), null)); - expect(relay.prepareChild(7, ["prepared-child"])).toBe(true); + expect(relay.prepareChild(7, routeCandidate(7, "prepared-child"), ["prepared-child"])).toBe(true); await vi.waitFor(() => expect(signals).toHaveLength(1)); const preparedConnection = FakePeerConnection.latest!; const preparedConnectionId = signals[0]!.connectionId; @@ -1257,9 +1229,9 @@ describe("ViewerRelay downstream ownership", () => { { sendSignal: () => true }, ); relay.setStream(createStream(createTrack("video", "rollback-video"), null)); - expect(relay.prepareChild(7, ["first-probe"])).toBe(true); + expect(relay.prepareChild(7, routeCandidate(7, "first-probe"), ["first-probe"])).toBe(true); const firstProbe = FakePeerConnection.latest!; - expect(relay.prepareChild(8, ["second-probe"])).toBe(true); + expect(relay.prepareChild(8, routeCandidate(8, "second-probe"), ["second-probe"])).toBe(true); const secondProbe = FakePeerConnection.latest!; expect(firstProbe.connectionState).toBe("closed"); relay.activateChildren(9, []); @@ -1267,17 +1239,17 @@ describe("ViewerRelay downstream ownership", () => { FakePeerConnection.offersFailing = 1; const instancesBeforeFailure = FakePeerConnection.instances.length; - expect(relay.prepareChild(10, ["failed-probe"])).toBe(true); + expect(relay.prepareChild(10, routeCandidate(10, "failed-probe"), ["failed-probe"])).toBe(true); const failedProbe = FakePeerConnection.latest!; await vi.waitFor(() => expect(failedProbe.connectionState).toBe("closed")); expect(FakePeerConnection.instances).toHaveLength(instancesBeforeFailure + 1); - expect(relay.prepareChild(11, ["session-probe"])).toBe(true); + expect(relay.prepareChild(11, routeCandidate(11, "session-probe"), ["session-probe"])).toBe(true); const sessionProbe = FakePeerConnection.latest!; relay.discardPreparedChild(); expect(sessionProbe.connectionState).toBe("closed"); - expect(relay.prepareChild(12, ["stopped-probe"])).toBe(true); + expect(relay.prepareChild(12, routeCandidate(12, "stopped-probe"), ["stopped-probe"])).toBe(true); const stoppedProbe = FakePeerConnection.latest!; relay.stop(); expect(stoppedProbe.connectionState).toBe("closed"); @@ -1297,7 +1269,7 @@ describe("ViewerRelay downstream ownership", () => { }, ); relay.setStream(createStream(createTrack("video", "initial-video"), null)); - expect(relay.prepareChild(7, ["prepared-child"])).toBe(true); + expect(relay.prepareChild(7, routeCandidate(7, "prepared-child"), ["prepared-child"])).toBe(true); await vi.waitFor(() => expect(signals).toHaveLength(1)); const preparedConnection = FakePeerConnection.latest!; const preparedConnectionId = signals[0]!.connectionId; @@ -1325,21 +1297,21 @@ describe("ViewerRelay downstream ownership", () => { { sendSignal: () => true }, ); relay.setStream(createStream(createTrack("video", "cap-video"), null)); - expect(relay.prepareChild(1, ["child-0"])).toBe(true); + expect(relay.prepareChild(1, routeCandidate(1, "child-0"), ["child-0"])).toBe(true); const preparedConnection = FakePeerConnection.latest!; preparedConnection.connectionState = "connected"; relay.activateChildren(1, ["child-0"]); expect(FakePeerConnection.activeCount).toBe(1); - expect(relay.prepareChild(2, ["child-0", "child-1"])).toBe(true); + expect(relay.prepareChild(2, routeCandidate(2, "child-1"), ["child-0", "child-1"])).toBe(true); FakePeerConnection.latest!.connectionState = "connected"; relay.activateChildren(2, ["child-0", "child-1"]); - expect(relay.prepareChild(3, ["child-0", "child-1", "child-2"])).toBe( + expect(relay.prepareChild(3, routeCandidate(3, "child-2"), ["child-0", "child-1", "child-2"])).toBe( true, ); FakePeerConnection.latest!.connectionState = "connected"; relay.activateChildren(3, ["child-0", "child-1", "child-2"]); expect( - relay.prepareChild(4, ["child-0", "child-1", "child-2", "child-3"]), + relay.prepareChild(4, routeCandidate(4, "child-3"), ["child-0", "child-1", "child-2", "child-3"]), ).toBe(false); expect(FakePeerConnection.activeCount).toBe(3); relay.dispose(); @@ -1363,7 +1335,13 @@ describe("ViewerRelay downstream ownership", () => { const activeConnection = FakePeerConnection.latest!; const activeConnectionId = signals[0]!.connectionId; activeConnection.connectionState = "connected"; - expect(relay.prepareChild(7, ["selected-child"])).toBe(true); + expect( + relay.prepareChild( + 7, + routeCandidate(7, "selected-child", "selected-turn", "selected-connection"), + ["selected-child"], + ), + ).toBe(true); expect(FakePeerConnection.latest).toBe(activeConnection); expect( diff --git a/tests/hybrid-media-router.test.ts b/tests/hybrid-media-router.test.ts new file mode 100644 index 0000000..790ead1 --- /dev/null +++ b/tests/hybrid-media-router.test.ts @@ -0,0 +1,393 @@ +import { describe, expect, it, vi } from "vitest"; + +import type { ServerMessage } from "../src/shared/protocol.ts"; +import { + HybridMediaRouter, + type AuthenticatedRouteParticipant, + type HybridAuthenticationState, +} from "../src/server/hybrid-media-router.ts"; +import { RoomStore } from "../src/server/room-store.ts"; +import { SfuResourceAdmission } from "../src/server/sfu-resource-admission.ts"; +import { TurnAllocationAdmission } from "../src/server/turn-allocation-admission.ts"; +import { FakeSfuRoomControl } from "./fake-sfu-room-control.ts"; + +const SHARE_GENERATION = "share_generation_12345678"; + +function createStore(maxViewersPerRoom = 20) { + return new RoomStore({ + ttlMs: 60_000, + maxRooms: 4, + maxViewersPerRoom, + }); +} + +function connectHost(store: RoomStore, room: ReturnType) { + const connected = store.connectParticipant({ + roomId: room.roomId, + role: "host", + token: room.hostToken, + clientId: "host_client_12345678", + sessionId: "host_session_12345678", + }); + return { + roomId: room.roomId, + role: "host" as const, + peerId: connected.peerId, + sessionId: "host_session_12345678", + }; +} + +function connectViewer( + store: RoomStore, + room: ReturnType, + suffix: string, +): AuthenticatedRouteParticipant { + const connected = store.connectParticipant({ + roomId: room.roomId, + role: "viewer", + ...(room.viewerGrant ? { viewerGrant: room.viewerGrant } : {}), + clientId: `viewer_client_${suffix}_12345678`, + sessionId: `viewer_session_${suffix}_12345678`, + }); + return { + roomId: room.roomId, + role: "viewer", + peerId: connected.peerId, + sessionId: `viewer_session_${suffix}_12345678`, + }; +} + +function complete( + router: HybridMediaRouter, + participant: AuthenticatedRouteParticipant, +): HybridAuthenticationState { + const state = router.connectParticipant(participant); + router.completeAuthentication(participant, state); + return state; +} + +function preparedFor( + sent: Map, + sessionId: string, +) { + return sent + .get(sessionId) + ?.findLast( + (message): message is Extract => + message.type === "route-update" && message.phase === "prepare", + ); +} + +function activeAfter( + sent: Map, + sessionId: string, + revision: number, +) { + return sent + .get(sessionId) + ?.findLast( + (message): message is Extract => + message.type === "route-update" && + message.phase === "active" && + message.revision > revision, + ); +} + +function harness( + endpointMediaCopyCapacity: 1 | 2 | 3, + withSfu = false, + withSelectedTurn = false, +) { + const store = createStore(); + const sent = new Map(); + const connections = new Map(); + const admission = withSfu + ? new SfuResourceAdmission({ ingressCapacity: 2, egressCapacity: 20 }) + : undefined; + const roomControl = withSfu ? new FakeSfuRoomControl() : undefined; + const turnAdmission = withSelectedTurn + ? new TurnAllocationAdmission({ capacity: 20 }) + : undefined; + const router = new HybridMediaRouter({ + roomStore: store, + endpointMediaCopyCapacity, + ...(admission && roomControl + ? { + sfuFallback: { + url: "wss://sfu.example.test", + admission, + roomControl, + tokenIssuer: { + async issueToken({ peerId }) { + return `token-${peerId}`; + }, + }, + }, + } + : {}), + ...(turnAdmission + ? { + selectedEdgeTurn: { + config: { + urls: ["turn:turn.example.test:3478?transport=udp"] as const, + sharedSecret: "t".repeat(32), + credentialTtlSeconds: 120, + allocationCapacity: 20, + }, + admission: turnAdmission, + }, + } + : {}), + sendToSession(sessionId, message) { + const messages = sent.get(sessionId) ?? []; + messages.push(message); + sent.set(sessionId, messages); + }, + getConnectionId(roomId, viewerPeerId) { + return connections.get(`${roomId}:${viewerPeerId}`); + }, + setConnectionId(roomId, viewerPeerId, connectionId) { + connections.set(`${roomId}:${viewerPeerId}`, connectionId); + }, + deleteConnectionId(roomId, viewerPeerId) { + connections.delete(`${roomId}:${viewerPeerId}`); + }, + getShareGeneration: () => SHARE_GENERATION, + }); + return { store, sent, admission, turnAdmission, router }; +} + +describe("HybridMediaRouter v7 runtime", () => { + it("orders exact prepare, commits only child proof, and rolls back above P", async () => { + const { store, sent, router } = harness(2); + const room = store.createRoom(); + const host = connectHost(store, room); + complete(router, host); + const first = connectViewer(store, room, "first"); + complete(router, first); + + await vi.waitFor(() => expect(preparedFor(sent, first.sessionId)).toBeDefined()); + const firstPrepare = preparedFor(sent, first.sessionId)!; + expect(firstPrepare.candidate).toMatchObject({ + childPeerId: first.peerId, + transport: "direct", + }); + expect(preparedFor(sent, host.sessionId)?.candidate).toEqual( + firstPrepare.candidate, + ); + + router.handleRouteReady(host, { + type: "route-ready", + revision: firstPrepare.revision, + phase: "prepare", + }); + expect(activeAfter(sent, first.sessionId, firstPrepare.revision)).toBeUndefined(); + router.handleRouteReady(first, { + type: "route-ready", + revision: firstPrepare.revision, + phase: "prepare", + }); + await vi.waitFor(() => + expect( + sent + .get(first.sessionId) + ?.findLast( + (message) => + message.type === "route-update" && + message.phase === "active" && + message.revision === firstPrepare.revision, + ), + ).toBeDefined(), + ); + + const second = connectViewer(store, room, "second"); + complete(router, second); + await vi.waitFor(() => expect(preparedFor(sent, second.sessionId)).toBeDefined()); + const secondPrepare = preparedFor(sent, second.sessionId)!; + router.handleRouteFailed(second, { + type: "route-failed", + revision: secondPrepare.revision, + phase: "prepare", + connectionId: secondPrepare.candidate.connectionId, + }); + await vi.waitFor(() => + expect(activeAfter(sent, second.sessionId, secondPrepare.revision)).toBeDefined(), + ); + expect(activeAfter(sent, second.sessionId, secondPrepare.revision)!.revision).toBeGreaterThan( + secondPrepare.revision, + ); + await router.close(); + }); + + it("creates one Host publication and reuses exact Viewer subscriptions", async () => { + const { store, sent, admission, router } = harness(1, true); + const room = store.createRoom(); + const host = connectHost(store, room); + complete(router, host); + const first = connectViewer(store, room, "sfu_first"); + complete(router, first); + const firstDirect = await vi.waitFor(() => { + const message = preparedFor(sent, first.sessionId); + expect(message).toBeDefined(); + return message!; + }); + router.handleRouteReady(first, { + type: "route-ready", + revision: firstDirect.revision, + phase: "prepare", + }); + + const second = connectViewer(store, room, "sfu_second"); + complete(router, second); + await vi.waitFor(() => + expect( + preparedFor(sent, first.sessionId)?.candidate.transport, + ).toBe("sfu"), + ); + const firstSfu = preparedFor(sent, first.sessionId)!; + router.handleRouteReady(first, { + type: "route-ready", + revision: firstSfu.revision, + phase: "prepare", + }); + + await vi.waitFor(() => + expect(preparedFor(sent, second.sessionId)?.candidate.transport).toBe("sfu"), + ); + const secondSfu = preparedFor(sent, second.sessionId)!; + router.handleRouteReady(second, { + type: "route-ready", + revision: secondSfu.revision, + phase: "prepare", + }); + await vi.waitFor(() => + expect(admission?.usage()).toEqual({ ingress: 1, egress: 2 }), + ); + const hostConfigs = sent + .get(host.sessionId) + ?.filter((message) => message.type === "sfu-config"); + expect(hostConfigs).toHaveLength(1); + await router.close(); + }); + + it("uses selected TURN only as the exact Viewer-parent edge transport", async () => { + const { store, sent, turnAdmission, router } = harness(1, true, true); + const room = store.createRoom(); + const host = connectHost(store, room); + complete(router, host); + const parent = connectViewer(store, room, "selected_parent"); + complete(router, parent); + await vi.waitFor(() => expect(preparedFor(sent, parent.sessionId)).toBeDefined()); + const parentPrepare = preparedFor(sent, parent.sessionId)!; + router.handleRouteReady(parent, { + type: "route-ready", + revision: parentPrepare.revision, + phase: "prepare", + }); + router.setViewerRelayCapacity(parent, 1); + + const child = connectViewer(store, room, "selected_child"); + complete(router, child); + await vi.waitFor(() => + expect(preparedFor(sent, child.sessionId)?.candidate.transport).toBe("direct"), + ); + const direct = preparedFor(sent, child.sessionId)!; + expect(direct.assignment.upstream).toEqual({ + kind: "peer", + peerId: parent.peerId, + }); + router.handleRouteReady(child, { + type: "route-ready", + revision: direct.revision, + phase: "prepare", + }); + + router.handleRouteFailed(child, { + type: "route-failed", + revision: direct.revision, + phase: "active", + connectionId: direct.candidate.connectionId, + }); + await vi.waitFor(() => + expect(preparedFor(sent, child.sessionId)?.candidate.transport).toBe( + "selected-turn", + ), + ); + const selected = preparedFor(sent, child.sessionId)!; + expect( + sent + .get(child.sessionId) + ?.some( + (message) => + message.type === "selected-edge-turn" && + message.edgeKind === "peer-selected" && + message.newConnectionId === selected.candidate.connectionId, + ), + ).toBe(true); + router.handleRouteReady(child, { + type: "route-ready", + revision: selected.revision, + phase: "prepare", + }); + await vi.waitFor(() => + expect(turnAdmission?.usage()).toEqual({ allocations: 1 }), + ); + await router.close(); + expect(turnAdmission?.usage()).toEqual({ allocations: 0 }); + }); + + it("uses selected TURN as one Host-SFU ingress after direct ingress fails", async () => { + const { store, sent, turnAdmission, router } = harness(1, true, true); + const room = store.createRoom(); + const host = connectHost(store, room); + complete(router, host); + const first = connectViewer(store, room, "ingress_first"); + complete(router, first); + await vi.waitFor(() => expect(preparedFor(sent, first.sessionId)).toBeDefined()); + const direct = preparedFor(sent, first.sessionId)!; + router.handleRouteReady(first, { + type: "route-ready", + revision: direct.revision, + phase: "prepare", + }); + + const waiting = connectViewer(store, room, "ingress_waiting"); + complete(router, waiting); + await vi.waitFor(() => + expect(preparedFor(sent, first.sessionId)?.candidate.transport).toBe("sfu"), + ); + const directIngress = preparedFor(sent, first.sessionId)!; + router.handleRouteFailed(first, { + type: "route-failed", + revision: directIngress.revision, + phase: "prepare", + connectionId: directIngress.candidate.connectionId, + }); + + await vi.waitFor(() => { + const prepared = preparedFor(sent, first.sessionId); + expect(prepared?.revision).toBeGreaterThan(directIngress.revision); + return prepared; + }); + const selectedIngress = preparedFor(sent, first.sessionId)!; + expect( + sent + .get(host.sessionId) + ?.some( + (message) => + message.type === "selected-edge-turn" && + message.edgeKind === "host-sfu-ingress" && + message.revision === selectedIngress.revision, + ), + ).toBe(true); + router.handleRouteReady(first, { + type: "route-ready", + revision: selectedIngress.revision, + phase: "prepare", + }); + await vi.waitFor(() => + expect(turnAdmission?.usage()).toEqual({ allocations: 1 }), + ); + await router.close(); + }); +}); diff --git a/tests/media-route-controller.test.ts b/tests/media-route-controller.test.ts deleted file mode 100644 index 4e78008..0000000 --- a/tests/media-route-controller.test.ts +++ /dev/null @@ -1,487 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { - MediaRouteController, - type PrepareMediaRouteInput, -} from "../src/server/media-route-controller.ts"; -import type { ParticipantRouteAssignment } from "../src/shared/protocol.ts"; - -const HOST = "host_12345678"; -const DIRECT_ROOT = "viewer_direct_12345678"; -const SFU_ROOT_A = "viewer_sfu_a_12345678"; -const SFU_ROOT_B = "viewer_sfu_b_12345678"; -const THIRD_CHILD = "viewer_third_12345678"; -const SFU_ROOT_C = "viewer_sfu_c_12345678"; -const GENERATION_A = "generation_a_12345678"; -const GENERATION_B = "generation_b_12345678"; - -function assignment( - upstream: ParticipantRouteAssignment["upstream"], - childPeerIds: string[] = [], - sfuPublicationGeneration: string | null = null, -): ParticipantRouteAssignment { - return { upstream, childPeerIds, sfuPublicationGeneration }; -} - -function directAssignments(): Map { - return new Map([ - [HOST, assignment({ kind: "none" }, [DIRECT_ROOT, SFU_ROOT_A])], - [DIRECT_ROOT, assignment({ kind: "peer", peerId: HOST })], - [SFU_ROOT_A, assignment({ kind: "peer", peerId: HOST })], - [SFU_ROOT_B, assignment({ kind: "none" })], - ]); -} - -function hybridPlan( - generation = GENERATION_A, - rootPeerId = SFU_ROOT_A, -): PrepareMediaRouteInput { - const otherRootPeerId = - rootPeerId === SFU_ROOT_A ? SFU_ROOT_B : SFU_ROOT_A; - return { - assignments: new Map([ - [ - HOST, - assignment({ kind: "none" }, [DIRECT_ROOT], generation), - ], - [DIRECT_ROOT, assignment({ kind: "peer", peerId: HOST })], - [rootPeerId, assignment({ kind: "sfu" })], - [otherRootPeerId, assignment({ kind: "none" })], - ]), - expectedParticipantIds: new Set([HOST, rootPeerId]), - sfuPublicationGeneration: generation, - sfuRootPeerIds: [rootPeerId], - }; -} - -function controller( - assignments = directAssignments(), -): MediaRouteController { - return new MediaRouteController({ hostPeerId: HOST, assignments }); -} - -function chainedAssignments(): Map { - return new Map([ - [HOST, assignment({ kind: "none" }, [DIRECT_ROOT])], - [ - DIRECT_ROOT, - assignment({ kind: "peer", peerId: HOST }, [SFU_ROOT_A]), - ], - [SFU_ROOT_A, assignment({ kind: "peer", peerId: DIRECT_ROOT })], - [SFU_ROOT_B, assignment({ kind: "none" })], - ]); -} - -describe("MediaRouteController", () => { - it("keeps SFU roots unique while charging one Host publication edge", () => { - const assignments = new Map([ - [ - HOST, - assignment({ kind: "none" }, [DIRECT_ROOT], GENERATION_A), - ], - [DIRECT_ROOT, assignment({ kind: "peer", peerId: HOST })], - [SFU_ROOT_A, assignment({ kind: "sfu" })], - [SFU_ROOT_B, assignment({ kind: "sfu" })], - ]); - const routes = new MediaRouteController({ - hostPeerId: HOST, - assignments, - sfuPublicationGeneration: GENERATION_A, - sfuRootPeerIds: [SFU_ROOT_A, SFU_ROOT_B], - }); - - expect(routes.getActiveRoute().sfu.rootPeerIds).toEqual([ - SFU_ROOT_A, - SFU_ROOT_B, - ]); - expect(routes.hostActiveMediaEdges()).toBe(2); - - const threeRootAssignments = new Map(assignments); - threeRootAssignments.set(SFU_ROOT_C, assignment({ kind: "sfu" })); - const threeRoots = new MediaRouteController({ - hostPeerId: HOST, - assignments: threeRootAssignments, - sfuPublicationGeneration: GENERATION_A, - sfuRootPeerIds: [SFU_ROOT_A, SFU_ROOT_B, SFU_ROOT_C], - }); - expect(threeRoots.getActiveRoute().sfu.rootPeerIds).toHaveLength(3); - expect(() => - new MediaRouteController({ - hostPeerId: HOST, - assignments, - sfuPublicationGeneration: GENERATION_A, - sfuRootPeerIds: [SFU_ROOT_A, SFU_ROOT_A], - }), - ).toThrow("SFU root participants must be unique"); - - const excessiveHostEdges = new Map(assignments); - excessiveHostEdges.set( - HOST, - assignment( - { kind: "none" }, - [DIRECT_ROOT, SFU_ROOT_C], - GENERATION_A, - ), - ); - excessiveHostEdges.set( - SFU_ROOT_C, - assignment({ kind: "peer", peerId: HOST }), - ); - expect( - () => - new MediaRouteController({ - hostPeerId: HOST, - assignments: excessiveHostEdges, - sfuPublicationGeneration: GENERATION_A, - sfuRootPeerIds: [SFU_ROOT_A, SFU_ROOT_B], - }), - ).toThrow("Endpoint steady media copy capacity exceeded"); - }); - - it("atomically commits only after every expected participant is ready", () => { - const routes = controller(); - const revision = routes.prepare(hybridPlan())!; - - expect(revision).toBe(1); - expect(routes.getActiveRoute().sfu.publicationGeneration).toBeNull(); - expect(routes.ready(HOST, revision, "prepare")).toBe(true); - expect(routes.ready(HOST, revision, "prepare")).toBe(false); - expect(routes.commit(revision)).toBe(false); - expect(routes.getActiveRoute().revision).toBe(0); - - expect(routes.ready(SFU_ROOT_A, revision, "prepare")).toBe(true); - expect(routes.getActiveRoute().sfu.publicationGeneration).toBeNull(); - expect(routes.commit(revision)).toBe(true); - expect(routes.getPendingRoute()).toBeUndefined(); - expect(routes.getActiveRoute()).toMatchObject({ - revision, - sfu: { - publicationGeneration: GENERATION_A, - rootPeerIds: [SFU_ROOT_A], - }, - }); - expect(routes.hostActiveMediaEdges()).toBe(2); - }); - - it("allows only one pending plan and ignores stale or wrong-phase messages", () => { - const routes = controller(); - const revision = routes.prepare(hybridPlan())!; - - expect(routes.prepare(hybridPlan(GENERATION_B, SFU_ROOT_B))).toBeUndefined(); - expect(routes.ready(HOST, revision - 1, "prepare")).toBe(false); - expect(routes.ready(HOST, revision, "active")).toBe(false); - expect(routes.commit(revision - 1)).toBe(false); - expect(routes.abort(revision - 1)).toBe(false); - expect(routes.getPendingRoute()?.route.revision).toBe(revision); - expect(routes.getActiveRoute().revision).toBe(0); - }); - - it("aborts to the unchanged active topology with a newer revision", () => { - const routes = controller(); - const before = routes.getActiveRoute(); - const preparedRevision = routes.prepare(hybridPlan())!; - - expect(routes.abort(preparedRevision)).toBe(true); - const after = routes.getActiveRoute(); - expect(after.revision).toBe(preparedRevision + 1); - expect(after.assignments).toEqual(before.assignments); - expect(after.sfu).toEqual(before.sfu); - expect(routes.getPendingRoute()).toBeUndefined(); - expect(routes.abort(preparedRevision)).toBe(false); - - expect(routes.prepare(hybridPlan())).toBe(after.revision + 1); - }); - - it("reconciles complete baseline snapshots without resetting revisions", () => { - const routes = controller(); - const reconciledRevision = routes.reconcileBaseline({ - assignments: chainedAssignments(), - }); - - expect(reconciledRevision).toBe(1); - expect(routes.getActiveRoute().revision).toBe(1); - expect( - routes.getActiveRoute().assignments.get(SFU_ROOT_A)?.upstream, - ).toEqual({ kind: "peer", peerId: DIRECT_ROOT }); - expect( - routes.reconcileBaseline({ assignments: chainedAssignments() }), - ).toBeUndefined(); - expect(routes.getActiveRoute().revision).toBe(1); - - const preparedRevision = routes.prepare(hybridPlan())!; - expect( - routes.reconcileBaseline({ assignments: directAssignments() }), - ).toBeUndefined(); - expect(routes.getActiveRoute().revision).toBe(1); - expect(routes.abort(preparedRevision)).toBe(true); - expect(routes.getActiveRoute().revision).toBe(3); - - expect( - routes.reconcileBaseline({ assignments: directAssignments() }), - ).toBe(4); - expect(routes.getActiveRoute().revision).toBe(4); - expect(routes.hostActiveMediaEdges()).toBe(2); - }); - - it("preserves active SFU state while reconciling peer topology", () => { - const activePlan = hybridPlan(GENERATION_A, SFU_ROOT_A); - const routes = new MediaRouteController({ - hostPeerId: HOST, - assignments: activePlan.assignments, - revision: 6, - sfuPublicationGeneration: GENERATION_A, - sfuRootPeerIds: [SFU_ROOT_A], - }); - const assignments = new Map(activePlan.assignments); - assignments.set( - DIRECT_ROOT, - assignment({ kind: "peer", peerId: HOST }, [SFU_ROOT_B]), - ); - assignments.set( - SFU_ROOT_B, - assignment({ kind: "peer", peerId: DIRECT_ROOT }), - ); - - expect(routes.reconcileBaseline({ assignments })).toBe(7); - expect(routes.getActiveRoute().sfu).toEqual({ - publicationGeneration: GENERATION_A, - rootPeerIds: [SFU_ROOT_A], - }); - expect(routes.hostActiveMediaEdges()).toBe(2); - }); - - it("never overlaps old and new SFU publication generations", () => { - const initialPlan = hybridPlan(GENERATION_A, SFU_ROOT_A); - const routes = new MediaRouteController({ - hostPeerId: HOST, - assignments: initialPlan.assignments, - revision: 4, - sfuPublicationGeneration: GENERATION_A, - sfuRootPeerIds: [SFU_ROOT_A], - }); - const preparedRevision = routes.prepare( - hybridPlan(GENERATION_B, SFU_ROOT_B), - )!; - - expect(routes.getActiveRoute().sfu).toEqual({ - publicationGeneration: GENERATION_A, - rootPeerIds: [SFU_ROOT_A], - }); - expect(routes.getPendingRoute()?.route.sfu).toEqual({ - publicationGeneration: GENERATION_B, - rootPeerIds: [SFU_ROOT_B], - }); - expect(routes.hostActiveMediaEdges()).toBe(2); - - routes.ready(HOST, preparedRevision, "prepare"); - routes.ready(SFU_ROOT_B, preparedRevision, "prepare"); - expect(routes.commit(preparedRevision)).toBe(true); - expect(routes.getActiveRoute().sfu).toEqual({ - publicationGeneration: GENERATION_B, - rootPeerIds: [SFU_ROOT_B], - }); - expect(routes.hostActiveMediaEdges()).toBe(2); - }); - - it("rejects host fanout above the configured two and inconsistent SFU ownership", () => { - const tooManyHostEdges = new Map(hybridPlan().assignments); - tooManyHostEdges.set( - HOST, - assignment( - { kind: "none" }, - [DIRECT_ROOT, SFU_ROOT_B], - GENERATION_A, - ), - ); - tooManyHostEdges.set( - SFU_ROOT_B, - assignment({ kind: "peer", peerId: HOST }), - ); - - expect(() => - new MediaRouteController({ - hostPeerId: HOST, - assignments: tooManyHostEdges, - sfuPublicationGeneration: GENERATION_A, - sfuRootPeerIds: [SFU_ROOT_A], - }), - ).toThrow("Endpoint steady media copy capacity exceeded"); - - const nonHostPublisher = directAssignments(); - nonHostPublisher.set( - DIRECT_ROOT, - assignment({ kind: "peer", peerId: HOST }, [], GENERATION_A), - ); - expect(() => controller(nonHostPublisher)).toThrow( - "Only the host may own an SFU publication generation", - ); - - const missingRoot = hybridPlan().assignments; - expect(() => - new MediaRouteController({ - hostPeerId: HOST, - assignments: missingRoot, - sfuPublicationGeneration: GENERATION_A, - sfuRootPeerIds: [SFU_ROOT_B], - }), - ).toThrow("Every SFU upstream must belong to the root allowlist"); - - const generationWithoutRoots = directAssignments(); - generationWithoutRoots.set( - HOST, - assignment( - { kind: "none" }, - [DIRECT_ROOT, SFU_ROOT_A], - GENERATION_A, - ), - ); - expect(() => - new MediaRouteController({ - hostPeerId: HOST, - assignments: generationWithoutRoots, - sfuPublicationGeneration: GENERATION_A, - }), - ).toThrow("SFU roots and publication generation must be active together"); - }); - - it("applies capacity three uniformly to Host and Viewer endpoints", () => { - const threeHostChildren = new Map([ - [ - HOST, - assignment( - { kind: "none" }, - [DIRECT_ROOT, SFU_ROOT_A, THIRD_CHILD], - ), - ], - [DIRECT_ROOT, assignment({ kind: "peer", peerId: HOST })], - [SFU_ROOT_A, assignment({ kind: "peer", peerId: HOST })], - [THIRD_CHILD, assignment({ kind: "peer", peerId: HOST })], - ]); - const hostAtCapacity = new MediaRouteController({ - hostPeerId: HOST, - assignments: threeHostChildren, - endpointMediaCopyCapacity: 3, - }); - expect(hostAtCapacity.hostActiveMediaEdges()).toBe(3); - - const viewerWithTwoChildren = new Map([ - [HOST, assignment({ kind: "none" }, [DIRECT_ROOT])], - [ - DIRECT_ROOT, - assignment( - { kind: "peer", peerId: HOST }, - [SFU_ROOT_A, SFU_ROOT_B], - ), - ], - [SFU_ROOT_A, assignment({ kind: "peer", peerId: DIRECT_ROOT })], - [SFU_ROOT_B, assignment({ kind: "peer", peerId: DIRECT_ROOT })], - ]); - expect( - new MediaRouteController({ - hostPeerId: HOST, - assignments: viewerWithTwoChildren, - endpointMediaCopyCapacity: 3, - }).getActiveRoute().assignments.get(DIRECT_ROOT)?.childPeerIds, - ).toHaveLength(2); - - const publicationAtHostLimit = new Map([ - [ - HOST, - assignment({ kind: "none" }, [DIRECT_ROOT], GENERATION_A), - ], - [DIRECT_ROOT, assignment({ kind: "peer", peerId: HOST })], - [SFU_ROOT_A, assignment({ kind: "sfu" })], - ]); - const allowed = new MediaRouteController({ - hostPeerId: HOST, - assignments: publicationAtHostLimit, - endpointMediaCopyCapacity: 3, - sfuPublicationGeneration: GENERATION_A, - sfuRootPeerIds: [SFU_ROOT_A], - }); - expect(allowed.hostActiveMediaEdges()).toBe(2); - - const publicationOverHostLimit = new Map(publicationAtHostLimit); - publicationOverHostLimit.set( - HOST, - assignment( - { kind: "none" }, - [DIRECT_ROOT, THIRD_CHILD, SFU_ROOT_B], - GENERATION_A, - ), - ); - publicationOverHostLimit.set( - THIRD_CHILD, - assignment({ kind: "peer", peerId: HOST }), - ); - publicationOverHostLimit.set( - SFU_ROOT_B, - assignment({ kind: "peer", peerId: HOST }), - ); - expect(() => - new MediaRouteController({ - hostPeerId: HOST, - assignments: publicationOverHostLimit, - endpointMediaCopyCapacity: 3, - sfuPublicationGeneration: GENERATION_A, - sfuRootPeerIds: [SFU_ROOT_A], - }), - ).toThrow("Endpoint steady media copy capacity exceeded"); - }); - - it("rejects reciprocal peer cycles", () => { - const assignments = new Map([ - [HOST, assignment({ kind: "none" })], - [ - SFU_ROOT_A, - assignment({ kind: "peer", peerId: SFU_ROOT_B }, [SFU_ROOT_B]), - ], - [ - SFU_ROOT_B, - assignment({ kind: "peer", peerId: SFU_ROOT_A }, [SFU_ROOT_A]), - ], - ]); - - expect(() => controller(assignments)).toThrow( - "Peer media route contains a cycle", - ); - }); - - it("preserves budgets and monotonic revisions across mixed sequences", () => { - const routes = controller(); - let previousRevision = routes.getActiveRoute().revision; - - for (let index = 0; index < 60; index += 1) { - const plan = - index % 2 === 0 - ? hybridPlan(GENERATION_A, SFU_ROOT_A) - : { - assignments: directAssignments(), - expectedParticipantIds: new Set([HOST]), - }; - const revision = routes.prepare(plan)!; - expect(revision).toBeGreaterThan(previousRevision); - expect(routes.prepare(plan)).toBeUndefined(); - expect(routes.ready(HOST, revision - 1, "prepare")).toBe(false); - expect(routes.commit(revision - 1)).toBe(false); - expect(routes.hostActiveMediaEdges()).toBeLessThanOrEqual(2); - - expect(routes.ready(HOST, revision, "prepare")).toBe(true); - expect(routes.ready(HOST, revision, "prepare")).toBe(false); - if (index % 3 === 0) { - expect(routes.abort(revision)).toBe(true); - } else { - if (plan.expectedParticipantIds.has(SFU_ROOT_A)) { - expect(routes.ready(SFU_ROOT_A, revision, "prepare")).toBe(true); - } - expect(routes.commit(revision)).toBe(true); - } - - const active = routes.getActiveRoute(); - expect(active.revision).toBeGreaterThan(previousRevision); - expect(routes.getPendingRoute()).toBeUndefined(); - expect(routes.hostActiveMediaEdges()).toBeLessThanOrEqual(2); - previousRevision = active.revision; - } - }); -}); diff --git a/tests/peer-relay-topology.test.ts b/tests/peer-relay-topology.test.ts deleted file mode 100644 index 6e3a70c..0000000 --- a/tests/peer-relay-topology.test.ts +++ /dev/null @@ -1,834 +0,0 @@ -import { describe, expect, it } from "vitest"; - -import { PeerRelayTopology } from "../src/server/peer-relay-topology.ts"; - -function connected(...peerIds: string[]): Set { - return new Set(peerIds); -} - -function addRelayViewer( - topology: PeerRelayTopology, - roomId: string, - peerId: string, - connectedPeerIds: ReadonlySet, - downstreamEdges: 1 | 2 | 3 = 1, -) { - const changes = topology.addViewer(roomId, peerId, connectedPeerIds); - return [ - ...changes, - ...topology.setViewerRelayCapacity( - roomId, - peerId, - downstreamEdges, - connectedPeerIds, - ), - ]; -} - -function expectValidTree( - topology: PeerRelayTopology, - roomId: string, - hostPeerId: string, - viewerPeerIds: readonly string[], - endpointMediaCopyCapacity = 2, -): number { - const hostAssignment = topology.getAssignment(roomId, hostPeerId)!; - expect(hostAssignment.parentPeerId).toBeNull(); - expect(hostAssignment.childPeerIds.length).toBeLessThanOrEqual( - endpointMediaCopyCapacity, - ); - - const allChildren = new Set(hostAssignment.childPeerIds); - let maximumDepth = 0; - for (const peerId of viewerPeerIds) { - const assignment = topology.getAssignment(roomId, peerId)!; - expect(assignment.childPeerIds.length).toBeLessThanOrEqual( - endpointMediaCopyCapacity, - ); - for (const childPeerId of assignment.childPeerIds) { - expect(allChildren.has(childPeerId)).toBe(false); - allChildren.add(childPeerId); - } - - let parentPeerId = assignment.parentPeerId; - const ancestors = new Set([peerId]); - let depth = 0; - while (parentPeerId) { - expect(ancestors.has(parentPeerId)).toBe(false); - ancestors.add(parentPeerId); - depth += 1; - parentPeerId = topology.getAssignment(roomId, parentPeerId)?.parentPeerId ?? null; - } - maximumDepth = Math.max(maximumDepth, depth); - } - expect(allChildren).toEqual(new Set(viewerPeerIds)); - return maximumDepth; -} - -describe("PeerRelayTopology", () => { - it("keeps viewers as leaves until a relay slot is explicitly advertised", () => { - const topology = new PeerRelayTopology(); - const peers = connected("host", "viewer-a", "viewer-b", "viewer-c"); - topology.setHost("room", "host", peers); - topology.addViewer("room", "viewer-a", peers); - topology.addViewer("room", "viewer-b", peers); - topology.addViewer("room", "viewer-c", peers); - - expect(topology.getAssignment("room", "viewer-c")?.parentPeerId).toBeNull(); - expect( - topology.setViewerRelayCapacity("room", "viewer-a", 0, peers), - ).toEqual([]); - - const changes = topology.setViewerRelayCapacity( - "room", - "viewer-a", - 1, - peers, - ); - expect(changes.map(({ peerId }) => peerId).sort()).toEqual([ - "viewer-a", - "viewer-c", - ]); - expect(topology.getAssignment("room", "viewer-a")).toEqual({ - parentPeerId: "host", - childPeerIds: ["viewer-c"], - }); - expect(topology.getAssignment("room", "viewer-b")?.parentPeerId).toBe( - "host", - ); - }); - - it("deterministically promotes an unassigned relay over the oldest zero-capacity host leaf", () => { - const topology = new PeerRelayTopology(); - const peers = connected("host", "viewer-a", "viewer-b", "viewer-c"); - topology.setHost("room", "host", peers); - for (const peerId of ["viewer-a", "viewer-b", "viewer-c"]) { - topology.addViewer("room", peerId, peers); - } - - const changes = topology.setViewerRelayCapacity( - "room", - "viewer-c", - 1, - peers, - { rescueUnassignedRelay: true }, - ); - - expect(changes.map(({ peerId }) => peerId)).toEqual([ - "host", - "viewer-a", - "viewer-c", - ]); - expect(topology.getAssignment("room", "host")).toEqual({ - parentPeerId: null, - childPeerIds: ["viewer-b", "viewer-c"], - }); - expect(topology.getAssignment("room", "viewer-c")).toEqual({ - parentPeerId: "host", - childPeerIds: ["viewer-a"], - }); - expect(topology.getAssignment("room", "viewer-a")).toEqual({ - parentPeerId: "viewer-c", - childPeerIds: [], - }); - expect( - expectValidTree( - topology, - "room", - "host", - ["viewer-a", "viewer-b", "viewer-c"], - ), - ).toBe(2); - - const afterRescue = topology.getAssignments("room"); - topology.setViewerRelayCapacity("room", "viewer-c", 0, peers); - expect( - topology.setViewerRelayCapacity( - "room", - "viewer-c", - 1, - peers, - { rescueUnassignedRelay: true }, - ), - ).toEqual([]); - expect(topology.getAssignments("room")).toEqual(afterRescue); - }); - - it("does not rescue admission through a host child that already owns a subtree", () => { - const topology = new PeerRelayTopology(); - const peers = connected("host", "a", "b", "c", "d", "candidate"); - topology.setHost("room", "host", peers); - topology.addViewer("room", "a", peers); - topology.addViewer("room", "b", peers); - topology.setViewerRelayCapacity("room", "a", 1, peers); - topology.setViewerRelayCapacity("room", "b", 1, peers); - topology.addViewer("room", "c", peers); - topology.addViewer("room", "d", peers); - topology.addViewer("room", "candidate", peers); - const before = topology.getAssignments("room"); - - expect( - topology.setViewerRelayCapacity( - "room", - "candidate", - 1, - peers, - { rescueUnassignedRelay: true }, - ), - ).toEqual([]); - expect(topology.getAssignments("room")).toEqual(before); - }); - - it("does not migrate healthy edges when a relay withdraws capacity", () => { - const topology = new PeerRelayTopology(); - const peers = connected("host", "viewer-a", "viewer-b", "viewer-c", "viewer-d"); - topology.setHost("room", "host", peers); - topology.addViewer("room", "viewer-a", peers); - topology.addViewer("room", "viewer-b", peers); - topology.setViewerRelayCapacity("room", "viewer-a", 1, peers); - topology.addViewer("room", "viewer-c", peers); - - expect( - topology.setViewerRelayCapacity("room", "viewer-a", 0, peers), - ).toEqual([]); - topology.addViewer("room", "viewer-d", peers); - - expect(topology.getAssignment("room", "viewer-c")?.parentPeerId).toBe( - "viewer-a", - ); - expect(topology.getAssignment("room", "viewer-d")?.parentPeerId).toBeNull(); - }); - - it("applies one deployment capacity to Host and Viewer endpoints", () => { - const peers = connected("host", "relay"); - const defaultTopology = new PeerRelayTopology(); - defaultTopology.setHost("default", "host", peers); - defaultTopology.addViewer("default", "relay", peers); - defaultTopology.setViewerRelayCapacity("default", "relay", 3, peers); - expect( - defaultTopology.getAdvertisedDownstreamCapacity("default", "host"), - ).toBe(2); - expect( - defaultTopology.getAdvertisedDownstreamCapacity("default", "relay"), - ).toBe(2); - - const expandedTopology = new PeerRelayTopology(3); - expandedTopology.setHost("expanded", "host", peers); - expandedTopology.addViewer("expanded", "relay", peers); - expandedTopology.setViewerRelayCapacity("expanded", "relay", 3, peers); - expect( - expandedTopology.getAdvertisedDownstreamCapacity("expanded", "host"), - ).toBe(3); - expect( - expandedTopology.getAdvertisedDownstreamCapacity("expanded", "relay"), - ).toBe(3); - - const tightenedTopology = new PeerRelayTopology(1); - tightenedTopology.setHost("tightened", "host", peers); - tightenedTopology.addViewer("tightened", "relay", peers); - tightenedTopology.setViewerRelayCapacity("tightened", "relay", 3, peers); - expect( - tightenedTopology.getAdvertisedDownstreamCapacity("tightened", "host"), - ).toBe(1); - expect( - tightenedTopology.getAdvertisedDownstreamCapacity("tightened", "relay"), - ).toBe(1); - }); - - it("does not reassign a failed edge through a viewer leaf", () => { - const topology = new PeerRelayTopology(); - const peers = connected("host", "viewer-a", "viewer-b", "viewer-c"); - topology.setHost("room", "host", peers); - topology.addViewer("room", "viewer-a", peers); - topology.addViewer("room", "viewer-b", peers); - topology.setViewerRelayCapacity("room", "viewer-a", 1, peers); - topology.addViewer("room", "viewer-c", peers); - - expect( - topology.reassignViewer( - "room", - "viewer-c", - peers, - new Set(["viewer-a"]), - ), - ).toBeUndefined(); - expect(topology.getAssignment("room", "viewer-c")?.parentPeerId).toBe( - "viewer-a", - ); - }); - - it("requires a reconnecting viewer to advertise capacity again", () => { - const topology = new PeerRelayTopology(); - const peers = connected("host", "viewer-a", "viewer-b", "viewer-c", "viewer-d"); - topology.setHost("room", "host", peers); - topology.addViewer("room", "viewer-a", peers); - topology.addViewer("room", "viewer-b", peers); - topology.setViewerRelayCapacity("room", "viewer-a", 1, peers); - topology.addViewer("room", "viewer-c", peers); - - topology.addViewer("room", "viewer-a", peers); - topology.addViewer("room", "viewer-d", peers); - - expect(topology.getAssignment("room", "viewer-c")?.parentPeerId).toBe( - "viewer-a", - ); - expect(topology.getAssignment("room", "viewer-d")?.parentPeerId).toBeNull(); - }); - - it("fills the shallowest stable slots with bounded fanout", () => { - const topology = new PeerRelayTopology(); - const roomId = "room"; - const host = "host"; - topology.setHost(roomId, host, connected(host)); - - const viewers = ["viewer-a", "viewer-b", "viewer-c", "viewer-d", "viewer-e"]; - for (const viewer of viewers) { - addRelayViewer(topology, roomId, viewer, connected(host, ...viewers)); - } - - expect(topology.getAssignment(roomId, host)).toEqual({ - parentPeerId: null, - childPeerIds: ["viewer-a", "viewer-b"], - }); - expect(topology.getAssignment(roomId, "viewer-a")).toEqual({ - parentPeerId: host, - childPeerIds: ["viewer-c"], - }); - expect(topology.getAssignment(roomId, "viewer-b")).toEqual({ - parentPeerId: host, - childPeerIds: ["viewer-d"], - }); - expect(topology.getAssignment(roomId, "viewer-c")).toEqual({ - parentPeerId: "viewer-a", - childPeerIds: ["viewer-e"], - }); - }); - - it("forms two bounded acyclic chains for eight viewers", () => { - const topology = new PeerRelayTopology(); - const roomId = "room"; - const hostPeerId = "host"; - const viewerPeerIds = Array.from( - { length: 8 }, - (_, index) => `viewer-${index + 1}`, - ); - topology.setHost(roomId, hostPeerId, connected(hostPeerId)); - for (const peerId of viewerPeerIds) { - addRelayViewer( - topology, - roomId, - peerId, - connected(hostPeerId, ...viewerPeerIds), - ); - } - - expect(expectValidTree(topology, roomId, hostPeerId, viewerPeerIds)).toBe(4); - }); - - it("uses capacity three uniformly in an eight-viewer tree", () => { - const topology = new PeerRelayTopology(3); - const roomId = "room"; - const hostPeerId = "host"; - const viewerPeerIds = Array.from( - { length: 8 }, - (_, index) => `viewer-${index + 1}`, - ); - const peers = connected(hostPeerId, ...viewerPeerIds); - topology.setHost(roomId, hostPeerId, peers); - - for (let index = 0; index < viewerPeerIds.length; index += 1) { - addRelayViewer( - topology, - roomId, - viewerPeerIds[index]!, - peers, - 3, - ); - expectValidTree( - topology, - roomId, - hostPeerId, - viewerPeerIds.slice(0, index + 1), - 3, - ); - } - - expect(topology.getAssignment(roomId, "viewer-1")?.childPeerIds).toEqual([ - "viewer-4", - "viewer-5", - "viewer-6", - ]); - expect(topology.getDownstreamCapacity(roomId, "viewer-1")).toBe(3); - expect(topology.getAssignment(roomId, "viewer-5")?.parentPeerId).toBe( - "viewer-1", - ); - expect( - expectValidTree(topology, roomId, hostPeerId, viewerPeerIds, 3), - ).toBe(2); - }); - - it("reattaches relay children within the shared endpoint capacity", () => { - const topology = new PeerRelayTopology(); - const peers = connected("host", "relay", "sibling", "child-a", "child-b"); - topology.setHost("room", "host", peers); - addRelayViewer(topology, "room", "relay", peers, 2); - addRelayViewer(topology, "room", "sibling", peers, 2); - addRelayViewer(topology, "room", "child-a", peers, 2); - addRelayViewer(topology, "room", "child-b", peers, 2); - expect(topology.getAssignment("room", "relay")?.childPeerIds).toEqual([ - "child-a", - "child-b", - ]); - expect(topology.getAssignment("room", "sibling")?.childPeerIds).toEqual([]); - - topology.removeViewer( - "room", - "relay", - connected("host", "sibling", "child-a", "child-b"), - ); - - expect(topology.getAssignment("room", "child-a")?.parentPeerId).toBe( - "host", - ); - expect(topology.getAssignment("room", "child-b")?.parentPeerId).toBe( - "sibling", - ); - expect( - expectValidTree( - topology, - "room", - "host", - ["sibling", "child-a", "child-b"], - 2, - ), - ).toBe(2); - }); - - it("uses the same depth-four bound for initial admission", () => { - const topology = new PeerRelayTopology(); - const roomId = "room"; - const peerIds = [ - "host", - "viewer-a", - "viewer-b", - "viewer-c", - "viewer-d", - "viewer-e", - "viewer-f", - ]; - const peers = connected(...peerIds); - topology.setHost(roomId, "host", peers); - topology.addViewer(roomId, "viewer-a", peers); - topology.setViewerRelayCapacity(roomId, "viewer-a", 1, peers); - topology.addViewer(roomId, "viewer-b", peers); - for (const peerId of ["viewer-c", "viewer-d", "viewer-e"]) { - topology.addViewer(roomId, peerId, peers); - topology.setViewerRelayCapacity(roomId, peerId, 1, peers); - } - topology.addViewer(roomId, "viewer-f", peers); - - expect(topology.getAssignment(roomId, "viewer-e")?.parentPeerId).toBe( - "viewer-d", - ); - expect(topology.getAssignment(roomId, "viewer-f")?.parentPeerId).toBeNull(); - expect( - expectValidTree( - topology, - roomId, - "host", - peerIds.slice(1, -1), - ), - ).toBe(4); - }); - - it("leaves a disconnected viewer subtree sticky during its grace window", () => { - const topology = new PeerRelayTopology(); - const roomId = "room"; - topology.setHost(roomId, "host", connected("host")); - addRelayViewer(topology, roomId, "viewer-a", connected("host", "viewer-a")); - addRelayViewer( - topology, - roomId, - "viewer-b", - connected("host", "viewer-a", "viewer-b"), - ); - addRelayViewer( - topology, - roomId, - "viewer-c", - connected("host", "viewer-a", "viewer-b", "viewer-c"), - ); - - addRelayViewer( - topology, - roomId, - "viewer-d", - connected("host", "viewer-b", "viewer-c", "viewer-d"), - ); - - expect(topology.getAssignment(roomId, "viewer-c")?.parentPeerId).toBe( - "viewer-a", - ); - expect(topology.getAssignment(roomId, "viewer-d")?.parentPeerId).toBe( - "viewer-b", - ); - }); - - it("reattaches only the removed viewer's direct child subtree entry", () => { - const topology = new PeerRelayTopology(); - const roomId = "room"; - topology.setHost(roomId, "host", connected("host")); - addRelayViewer(topology, roomId, "viewer-a", connected("host", "viewer-a")); - addRelayViewer( - topology, - roomId, - "viewer-b", - connected("host", "viewer-a", "viewer-b"), - ); - addRelayViewer( - topology, - roomId, - "viewer-c", - connected("host", "viewer-a", "viewer-b", "viewer-c"), - ); - addRelayViewer( - topology, - roomId, - "viewer-d", - connected("host", "viewer-a", "viewer-b", "viewer-c", "viewer-d"), - ); - addRelayViewer( - topology, - roomId, - "viewer-e", - connected( - "host", - "viewer-a", - "viewer-b", - "viewer-c", - "viewer-d", - "viewer-e", - ), - ); - - const changes = topology.removeViewer( - roomId, - "viewer-a", - connected("host", "viewer-b", "viewer-c", "viewer-d", "viewer-e"), - ); - - expect(topology.getAssignment(roomId, "host")?.childPeerIds).toEqual([ - "viewer-b", - "viewer-c", - ]); - expect(topology.getAssignment(roomId, "viewer-c")?.parentPeerId).toBe( - "host", - ); - expect(topology.getAssignment(roomId, "viewer-e")?.parentPeerId).toBe( - "viewer-c", - ); - expect(changes.map((change) => change.peerId).sort()).toEqual([ - "host", - "viewer-c", - ]); - }); - - it("waits to reattach an offline subtree root until it reconnects", () => { - const topology = new PeerRelayTopology(); - const roomId = "room"; - topology.setHost(roomId, "host", connected("host")); - addRelayViewer(topology, roomId, "viewer-a", connected("host", "viewer-a")); - addRelayViewer( - topology, - roomId, - "viewer-b", - connected("host", "viewer-a", "viewer-b"), - ); - addRelayViewer( - topology, - roomId, - "viewer-c", - connected("host", "viewer-a", "viewer-b", "viewer-c"), - ); - - topology.removeViewer( - roomId, - "viewer-a", - connected("host", "viewer-b"), - ); - - expect(topology.getAssignment(roomId, "viewer-c")?.parentPeerId).toBeNull(); - expect(topology.getAssignment(roomId, "host")?.childPeerIds).toEqual([ - "viewer-b", - ]); - - const reconnectChanges = addRelayViewer( - topology, - roomId, - "viewer-c", - connected("host", "viewer-b", "viewer-c"), - ); - - expect(topology.getAssignment(roomId, "viewer-c")?.parentPeerId).toBe( - "host", - ); - expect(reconnectChanges.map((change) => change.peerId).sort()).toEqual([ - "host", - "viewer-c", - ]); - }); - - it("assigns connected pending viewers when removals make slots reachable", () => { - const topology = new PeerRelayTopology(); - const roomId = "room"; - topology.setHost(roomId, "host", connected("host")); - for (const peerId of ["a", "b", "c", "d"]) { - addRelayViewer( - topology, - roomId, - peerId, - connected("host", "a", "b", "c", "d"), - ); - } - - addRelayViewer(topology, roomId, "e", connected("host", "c", "d", "e")); - expect(topology.getAssignment(roomId, "e")?.parentPeerId).toBeNull(); - - topology.removeViewer(roomId, "a", connected("host", "c", "d", "e")); - topology.removeViewer(roomId, "b", connected("host", "c", "d", "e")); - - expect(topology.getAssignment(roomId, "e")?.parentPeerId).toBe("c"); - expect(topology.getAssignment(roomId, "c")?.childPeerIds).toEqual(["e"]); - }); - - it("keeps pending viewers unassigned until a host arrives", () => { - const topology = new PeerRelayTopology(); - addRelayViewer(topology, "room", "viewer-a", connected("viewer-a")); - expect(topology.getAssignment("room", "viewer-a")?.parentPeerId).toBeNull(); - - topology.setHost( - "room", - "host", - connected("host", "viewer-a"), - ); - expect(topology.getAssignment("room", "viewer-a")?.parentPeerId).toBe( - "host", - ); - }); - - it("updates only direct root parents when the host identity changes", () => { - const topology = new PeerRelayTopology(); - topology.setHost("room", "host-a", connected("host-a")); - addRelayViewer( - topology, - "room", - "viewer-a", - connected("host-a", "viewer-a"), - ); - addRelayViewer( - topology, - "room", - "viewer-b", - connected("host-a", "viewer-a", "viewer-b"), - ); - addRelayViewer( - topology, - "room", - "viewer-c", - connected("host-a", "viewer-a", "viewer-b", "viewer-c"), - ); - - topology.setHost( - "room", - "host-b", - connected("host-b", "viewer-a", "viewer-b", "viewer-c"), - ); - - expect(topology.getAssignment("room", "viewer-a")?.parentPeerId).toBe( - "host-b", - ); - expect(topology.getAssignment("room", "viewer-b")?.parentPeerId).toBe( - "host-b", - ); - expect(topology.getAssignment("room", "viewer-c")?.parentPeerId).toBe( - "viewer-a", - ); - }); - - it("reassigns a failed edge deterministically without entering its subtree", () => { - const topology = new PeerRelayTopology(); - const peers = ["host", "a", "b", "c", "d", "e"]; - topology.setHost("room", "host", connected(...peers)); - for (const peerId of peers.slice(1)) { - addRelayViewer(topology, "room", peerId, connected(...peers)); - } - - expect(topology.getAssignment("room", "e")?.parentPeerId).toBe("c"); - const changes = topology.reassignViewer( - "room", - "e", - connected(...peers), - new Set(["c"]), - 4, - ); - - expect(changes?.map(({ peerId }) => peerId).sort()).toEqual([ - "c", - "d", - "e", - ]); - expect(topology.getAssignment("room", "e")?.parentPeerId).toBe("d"); - expect(expectValidTree(topology, "room", "host", peers.slice(1))).toBe(3); - }); - - it("plans a bounded reparent without mutating until the exact parent commits", () => { - const topology = new PeerRelayTopology(); - const peers = ["host", "a", "b", "c", "d", "e"]; - topology.setHost("room", "host", connected(...peers)); - for (const peerId of peers.slice(1)) { - addRelayViewer(topology, "room", peerId, connected(...peers)); - } - const before = topology.getAssignments("room"); - const excluded = new Set(["c"]); - - expect( - topology.findViewerReassignmentParent( - "room", - "e", - connected(...peers), - excluded, - 4, - ), - ).toBe("d"); - expect(topology.getAssignments("room")).toEqual(before); - expect( - topology.reassignViewer( - "room", - "e", - connected(...peers), - excluded, - 4, - "b", - ), - ).toBeUndefined(); - expect(topology.getAssignments("room")).toEqual(before); - expect( - topology.reassignViewer( - "room", - "e", - connected(...peers), - excluded, - 4, - "d", - ), - ).toBeDefined(); - expect(topology.getAssignment("room", "e")?.parentPeerId).toBe("d"); - }); - - it("does not mutate when every bounded reparent candidate is excluded", () => { - const topology = new PeerRelayTopology(); - topology.setHost("room", "host", connected("host", "a", "b", "c")); - for (const peerId of ["a", "b", "c"]) { - addRelayViewer( - topology, - "room", - peerId, - connected("host", "a", "b", "c"), - ); - } - const before = topology.getAssignments("room"); - - expect( - topology.reassignViewer( - "room", - "c", - connected("host", "a", "b", "c"), - new Set(["host", "a", "b"]), - 4, - ), - ).toBeUndefined(); - expect(topology.getAssignments("room")).toEqual(before); - }); - - it("rejects a reparent whose deepest descendant would exceed max depth", () => { - const topology = new PeerRelayTopology(); - const peers = ["host", "a", "b", "c", "d", "e", "f", "g", "h"]; - topology.setHost("room", "host", connected(...peers)); - for (const peerId of peers.slice(1)) { - addRelayViewer(topology, "room", peerId, connected(...peers)); - } - const before = topology.getAssignments("room"); - - expect(topology.getAssignment("room", "g")?.parentPeerId).toBe("e"); - expect( - topology.reassignViewer( - "room", - "g", - connected(...peers), - new Set(["e"]), - 4, - ), - ).toBeUndefined(); - expect(topology.getAssignments("room")).toEqual(before); - }); - - it("does not attach a pending subtree beyond the initial-admission depth bound", () => { - const topology = new PeerRelayTopology(); - const initialPeers = ["host", "a", "b", "c", "d", "e", "f", "g", "h"]; - topology.setHost("room", "host", connected(...initialPeers)); - for (const peerId of initialPeers.slice(1)) { - addRelayViewer(topology, "room", peerId, connected(...initialPeers)); - } - expect(topology.getAssignment("room", "e")).toMatchObject({ - parentPeerId: "c", - childPeerIds: ["g"], - }); - - topology.removeViewer( - "room", - "c", - connected("host", "a", "b", "d", "f", "h"), - ); - topology.removeViewer( - "room", - "h", - connected("host", "a", "b", "d", "f"), - ); - topology.addViewer( - "room", - "x", - connected("host", "a", "b", "d", "f", "x"), - ); - - topology.addViewer( - "room", - "e", - connected("host", "a", "b", "d", "e", "f", "g", "x"), - ); - - expect(topology.getAssignment("room", "e")?.parentPeerId).toBeNull(); - expect(topology.getAssignment("room", "g")?.parentPeerId).toBe("e"); - expect(topology.getAssignment("room", "f")?.childPeerIds).toEqual([]); - }); - - it("drops all assignments when a room is deleted", () => { - const topology = new PeerRelayTopology(); - topology.setHost("room", "host-a", connected("host-a")); - addRelayViewer( - topology, - "room", - "viewer-a", - connected("host-a", "viewer-a"), - ); - - topology.deleteRoom("room"); - expect(topology.getAssignment("room", "host-a")).toBeUndefined(); - expect(topology.getAssignment("room", "viewer-a")).toBeUndefined(); - - topology.setHost("room", "host-b", connected("host-b")); - expect(topology.getAssignment("room", "host-b")).toEqual({ - parentPeerId: null, - childPeerIds: [], - }); - }); -}); diff --git a/tests/protocol.test.ts b/tests/protocol.test.ts index 152ac78..61929fc 100644 --- a/tests/protocol.test.ts +++ b/tests/protocol.test.ts @@ -76,7 +76,7 @@ const parentEdgeQualityEvidence = { } as const; describe("client signaling protocol", () => { - it("keeps the Native sender on the same signaling version", () => { + it("keeps executable senders outside the Browser v7 release", () => { const nativeWire = readFileSync( join( import.meta.dirname, @@ -85,9 +85,8 @@ describe("client signaling protocol", () => { "utf8", ); - expect(nativeWire).toMatch( - new RegExp(`signalingProtocol\\s*=\\s*"${SIGNALING_PROTOCOL}"`), - ); + expect(SIGNALING_PROTOCOL).toBe("screener-v7"); + expect(nativeWire).toMatch(/signalingProtocol\s*=\s*"screener-v6"/); }); it("keeps signaling challenges strict and sequence-only", () => { @@ -730,13 +729,7 @@ describe("client signaling protocol", () => { }, ); - it("accepts bounded route acknowledgements, failures, and SFU recovery events", () => { - expect( - clientMessageSchema.safeParse({ - type: "sfu-reselection-ready", - revision: 7, - }).success, - ).toBe(true); + it("accepts bounded route acknowledgements and failures", () => { expect( clientMessageSchema.safeParse({ type: "route-ready", @@ -791,12 +784,6 @@ describe("client signaling protocol", () => { revision: 1.5, }).success, ).toBe(false); - expect( - clientMessageSchema.safeParse({ - type: "sfu-reselection-ready", - revision: -1, - }).success, - ).toBe(false); }); }); @@ -1107,6 +1094,11 @@ describe("server signaling protocol", () => { revision: 9, phase: "prepare", assignment, + candidate: { + childPeerId: "child_12345678", + connectionId: "connection_12345678", + transport: "direct", + }, }).success, ).toBe(true); expect( diff --git a/tests/room-route-controller.test.ts b/tests/room-route-controller.test.ts index 074846f..5721b6a 100644 --- a/tests/room-route-controller.test.ts +++ b/tests/room-route-controller.test.ts @@ -120,6 +120,34 @@ function peerEdge( } describe("RoomRouteController", () => { + it.each([1, 2, 3] as const)( + "admits 20 Viewers through one bounded event-driven graph at C=%i", + (capacity) => { + const routes = controller(capacity); + const viewerPeerIds = Array.from( + { length: 20 }, + (_, index) => `viewer_${String(index).padStart(2, "0")}_12345678`, + ); + viewerPeerIds.forEach((peerId, index) => { + addViewer(routes, peerId, capacity); + const operation = routes.reconcile(index * 2).operation!; + expect(operation.childPeerId).toBe(peerId); + commitCurrent(routes, index * 2 + 1, `${peerId}_connection`); + }); + const snapshot = routes.snapshot(); + expect(snapshot.upstreamByViewer).toHaveLength(20); + for (const parentPeerId of [HOST, ...viewerPeerIds]) { + const childCount = [...snapshot.upstreamByViewer.values()].filter( + (edge) => + edge.kind === "peer" && + edge.physicalActive && + edge.parentPeerId === parentPeerId, + ).length; + expect(childCount).toBeLessThanOrEqual(capacity); + } + }, + ); + it.each([1, 2, 3] as const)( "builds a deterministic acyclic route without a depth cap at C=%i", (capacity) => { @@ -217,6 +245,7 @@ describe("RoomRouteController", () => { connectionId: "candidate_1", }, 1_001); expect(failed.accepted).toBe(true); + expect(failed.activeRevision).toBeGreaterThan(first.current!.revision); expect(failed.exhausted).toBeUndefined(); expect(failed.released).toEqual([]); expect(routes.snapshot().operation).toMatchObject({ @@ -444,6 +473,7 @@ describe("RoomRouteController", () => { generation: "publication_1", connectionId: "publication:publication_1", })).toBe(true); + routes.touchExternalFacts(); const repair = routes.reconcile(2).operation!; expect(repair.childPeerId).toBe(A); expect(repair.candidates[0]?.tuple).toEqual({ @@ -487,11 +517,12 @@ describe("RoomRouteController", () => { connectionId: "a_sfu_2", }, 4); expect(repaired.accepted).toBe(true); - expect(repaired.released).toEqual([ + expect(repaired.released).toHaveLength(3); + expect(repaired.released).toEqual(expect.arrayContaining([ "publication_overlap", "a_subscription", "publication_resource", - ]); + ])); expect(routes.snapshot().hostPublication).toMatchObject({ generation: "publication_2", usable: true, @@ -599,6 +630,7 @@ describe("RoomRouteController", () => { generation: "publication_1", connectionId: "publication:publication_1", })).toBe(true); + routes.touchExternalFacts(); const operation = routes.reconcile(0).operation!; expect(operation.candidates[0]).toMatchObject({ tuple: { kind: "sfu", publication: "replace", ingress: "direct" }, @@ -623,12 +655,13 @@ describe("RoomRouteController", () => { connectionId: "a_sfu_2", }, 2); expect(settled.accepted).toBe(true); - expect(settled.released).toEqual([ + expect(settled.released).toHaveLength(4); + expect(settled.released).toEqual(expect.arrayContaining([ "publication_overlap", "subscription_a", "subscription_b", "publication_resource_1", - ]); + ])); expect(routes.snapshot().upstreamByViewer.get(A)).toMatchObject({ publicationGeneration: "publication_2", }); @@ -902,4 +935,77 @@ describe("RoomRouteController", () => { resource: "b_turn_resource", }); }); + + it("retries an exact failed tuple only after a new external fact", () => { + const routes = controller(1); + addViewer(routes, A, 0); + routes.hydrateEdge(A, peerEdge(HOST, "failed_direct")); + expect(routes.invalidateEdge({ + childPeerId: A, + childSessionId: `${A}_session`, + parentSessionId: "host_session", + routeRevision: 0, + connectionId: "failed_direct", + })).toBe(true); + expect(routes.reconcile(0).operation).toBeUndefined(); + expect(routes.snapshot().upstreamByViewer.get(A)).toMatchObject({ + usable: false, + physicalActive: false, + }); + + routes.touchExternalFacts(); + expect(routes.reconcile(1).operation?.candidates[0]?.tuple).toEqual({ + kind: "peer", + parentPeerId: HOST, + transport: "direct", + }); + }); + + it("appends one ordinary restore candidate after a healthy bounded gap", () => { + const routes = controller(3, { sfuEnabled: true }); + for (const [peerId, connectionId] of [ + [A, "a_direct"], + [B, "b_direct"], + [C, "c_direct"], + ] as const) { + addViewer(routes, peerId, 0); + routes.hydrateEdge(peerId, peerEdge(HOST, connectionId)); + } + addViewer(routes, D, 0); + const operation = routes.reconcile(0).operation!; + expect(operation.childPeerId).toBe(C); + expect(operation.candidates[0]?.endpointTransition.kind).toBe("bounded-gap"); + const gap = routes.retireCurrentCandidateProducer(cursorGuard(operation), 1); + expect(gap.accepted).toBe(true); + const tuples = routes.snapshot().operation?.candidates.map((plan) => plan.tuple); + expect(tuples?.at(-1)).toEqual({ + kind: "peer", + parentPeerId: HOST, + transport: "direct", + }); + }); + + it("does not promote when synchronous admission commit fails", () => { + const routes = controller(1); + addViewer(routes, A, 0); + const operation = routes.reconcile(0).operation!; + const prepared = beginCandidate(routes, { + nowMs: 1, + connectionId: "candidate_connection", + reservation: { kind: "direct" }, + }).operation!; + const settled = routes.candidateReady( + { + childPeerId: operation.childPeerId, + childSessionId: operation.childSessionId, + revision: prepared.current!.revision, + connectionId: "candidate_connection", + }, + 2, + () => false, + ); + expect(settled.accepted).toBe(false); + expect(settled.activeRevision).toBeGreaterThan(prepared.current!.revision); + expect(routes.snapshot().upstreamByViewer.has(A)).toBe(false); + }); }); diff --git a/tests/route-transition.test.ts b/tests/route-transition.test.ts index 057013f..5e2a051 100644 --- a/tests/route-transition.test.ts +++ b/tests/route-transition.test.ts @@ -72,9 +72,16 @@ function createFakePublisher(log: string[], label: string) { } type SubscriberEvents = Parameters< - NonNullable[0]["createSubscriber"]> + NonNullable[1]["createSubscriber"]> >[0]; +const candidate = ( + revision: number, + childPeerId = "viewer_12345678", + transport: "direct" | "selected-turn" | "sfu" = "direct", + connectionId = `candidate_${revision}_12345678`, +) => ({ childPeerId, connectionId, transport }); + function createFakeSubscriber(events: SubscriberEvents, log: string[], label: string) { return { events, @@ -106,7 +113,12 @@ describe("minimal route transition contracts", () => { const oldToken = route.token()!; expect(route.markMediaActive(oldToken)).toBe(true); - route.accept({ revision: 2, phase: "prepare", assignment: nextAssignment }); + route.accept({ + revision: 2, + phase: "prepare", + assignment: nextAssignment, + candidate: candidate(2), + }); expect(route.getMediaAssignment()).toEqual(oldAssignment); expect(route.markMediaActive(route.token()!)).toBe(false); @@ -121,7 +133,7 @@ describe("minimal route transition contracts", () => { it("keeps duplicate peer prepare idempotent and releases it before takeover", async () => { const prepared: Array<{ parentPeerId: string | null; revision?: number }> = []; const activatePeer = vi.fn(() => true); - const route = new ViewerSfuRoute({ + const route = new ViewerSfuRoute("viewer_12345678", { activatePeer, preparePeer: (assignment, revision) => prepared.push({ @@ -147,6 +159,7 @@ describe("minimal route transition contracts", () => { revision: 2, phase: "prepare", assignment: nextAssignment, + candidate: candidate(2), }), ).toBe("accepted"); expect(prepared).toEqual([ @@ -158,6 +171,7 @@ describe("minimal route transition contracts", () => { revision: 2, phase: "prepare", assignment: nextAssignment, + candidate: candidate(2), }), ).toBe("duplicate"); expect(prepared).toEqual([ @@ -168,6 +182,7 @@ describe("minimal route transition contracts", () => { revision: 3, phase: "prepare", assignment: peerAssignment("third-parent"), + candidate: candidate(3), }); expect(prepared.slice(-2)).toEqual([ { parentPeerId: null }, @@ -178,6 +193,7 @@ describe("minimal route transition contracts", () => { revision: 4, phase: "prepare", assignment: viewerSfuAssignment(), + candidate: candidate(4, "viewer_12345678", "sfu"), }); expect(prepared.at(-1)).toEqual({ parentPeerId: null }); }); @@ -257,6 +273,7 @@ describe("minimal route transition contracts", () => { revision: 2, phase: "prepare", assignment: hostAssignment("publication-b"), + candidate: candidate(2, "viewer_12345678", "sfu"), }); await route.acceptConfig(sfuConfig(2)); expect(publishers[1]?.activate).toHaveBeenCalledOnce(); @@ -291,8 +308,13 @@ describe("minimal route transition contracts", () => { revision: 2, phase: "prepare", assignment: hostAssignment("publication-b"), + candidate: candidate( + 2, + "viewer_12345678", + "sfu", + "selected-connection", + ), }); - await route.acceptConfig(sfuConfig(2)); expect( route.startSelectedEdgeTurn({ type: "selected-edge-turn", @@ -310,10 +332,10 @@ describe("minimal route transition contracts", () => { }, }), ).toBe(true); + await route.acceptConfig(sfuConfig(2)); - await vi.waitFor(() => expect(publishers).toHaveLength(2)); - expect(publishers[0]?.disconnect).toHaveBeenCalled(); - expect(publishers[1]?.connect).toHaveBeenCalledWith({ + await vi.waitFor(() => expect(publishers).toHaveLength(1)); + expect(publishers[0]?.connect).toHaveBeenCalledWith({ url: "wss://sfu.example.test", token: "token-2", rtcConfig: { @@ -334,7 +356,7 @@ describe("minimal route transition contracts", () => { const messages: ClientMessage[] = []; const streams: MediaStream[] = []; const subscribers: ReturnType[] = []; - const route = new ViewerSfuRoute({ + const route = new ViewerSfuRoute("viewer_12345678", { activatePeer: () => true, reconcileSfuChildren: () => undefined, onSfuStream: (stream) => streams.push(stream), @@ -370,6 +392,7 @@ describe("minimal route transition contracts", () => { revision: 2, phase: "prepare", assignment: viewerSfuAssignment(), + candidate: candidate(2, "viewer_12345678", "sfu"), }); await route.acceptConfig(sfuConfig(2)); const pendingStream = {} as MediaStream; @@ -403,7 +426,7 @@ describe("minimal route transition contracts", () => { it("discards a paused pending subscriber and ignores its later evidence", async () => { const messages: ClientMessage[] = []; let subscriber!: ReturnType; - const route = new ViewerSfuRoute({ + const route = new ViewerSfuRoute("viewer_12345678", { activatePeer: () => true, reconcileSfuChildren: () => undefined, onSfuStream: () => undefined, @@ -421,6 +444,7 @@ describe("minimal route transition contracts", () => { revision: 3, phase: "prepare", assignment: viewerSfuAssignment(), + candidate: candidate(3, "viewer_12345678", "sfu"), }); await route.acceptConfig(sfuConfig(3)); route.setPaused(true); diff --git a/tests/server-signal.test.ts b/tests/server-signal.test.ts index dd19424..ed3105e 100644 --- a/tests/server-signal.test.ts +++ b/tests/server-signal.test.ts @@ -1,19 +1,14 @@ import { connect } from "node:net"; -import { createServer } from "node:http"; import { afterEach, describe, expect, it, vi } from "vitest"; import WebSocket from "ws"; import { - MAX_PARENT_EDGE_QUALITY_EVIDENCE_BYTES, MAX_SIGNAL_BYTES, - MAX_VIEWER_QUALITY_EVIDENCE_BYTES, SIGNALING_PROTOCOL, decodeServerMessage, - type ParentEdgeQualityProof, type Role, type ServerMessage, - type ViewerQualityEvidenceMetrics, } from "../src/shared/protocol.ts"; import { createScreenerServer, @@ -22,40 +17,8 @@ import { import type { ServerConfig } from "../src/server/config.ts"; import { RoomDatabase } from "../src/server/room-database.ts"; import { RoomStore, type CreatedRoom } from "../src/server/room-store.ts"; -import { SignalingServer } from "../src/server/signaling.ts"; -import type { SfuTokenIssuer } from "../src/server/livekit-token.ts"; -import { HybridMediaRouter } from "../src/server/hybrid-media-router.ts"; -import { SfuResourceAdmission } from "../src/server/sfu-resource-admission.ts"; -import { managedSfuRoomName } from "../src/server/sfu-room-control.ts"; -import { TurnAllocationAdmission } from "../src/server/turn-allocation-admission.ts"; -import { FakeSfuRoomControl } from "./fake-sfu-room-control.ts"; const allowedOrigin = "http://allowed.test"; -const TEST_SFU_INGRESS_CAPACITY = 10; -const TEST_SFU_EGRESS_CAPACITY = 16; -const TEST_TURN_ALLOCATION_CAPACITY = 8; -const defaultQualitySettings = { - resolution: "1080p", - maxFramerate: 60, - maxBitrate: 8_000_000, - degradationPreference: "balanced", - videoCodec: "automatic", - screenAudioQuality: "music", -} as const; -const balancedQualitySettings = { - resolution: "1080p", - maxFramerate: 30, - maxBitrate: 5_000_000, - degradationPreference: "balanced", - screenAudioQuality: "saver", -} as const; -const lowQualitySettings = { - resolution: "720p", - maxFramerate: 30, - maxBitrate: 3_000_000, - degradationPreference: "maintain-resolution", - screenAudioQuality: "very-high", -} as const; let runningServer: ScreenerServer | undefined; afterEach(async () => { @@ -149,9 +112,6 @@ interface SignalHarness { roomStore: RoomStore; room: CreatedRoom; database?: RoomDatabase; - sfuAdmission?: SfuResourceAdmission; - turnAdmission?: TurnAllocationAdmission; - sfuRoomControl?: FakeSfuRoomControl; } function testConfig(): ServerConfig { @@ -229,540 +189,6 @@ async function startHarness( }; } -function viewerQualityEvidence( - connectionId: string, - routeRevision: number, - sequence: number, - bitrateKbps = 7_500, -) { - return { - type: "viewer-quality-evidence", - guard: { connectionId, routeRevision }, - sequence, - windowMs: 2_000, - metrics: { - width: 1_920, - height: 1_080, - framesPerSecond: 60, - bitrateKbps, - packetsReceivedDelta: 1_500, - packetsLostDelta: 2, - jitterMs: 3.5, - framesDecodedDelta: 120, - framesDroppedDelta: 1, - decodeMsPerFrame: 2.4, - freezeCountDelta: 0, - freezeDurationMsDelta: 0, - codec: "video/H264", - codecProfile: "profile-level-id=42e01f", - codecParameters: - "packetization-mode=1; level-asymmetry-allowed=1", - }, - } as const; -} - -function viewerQualityEvidenceWithMetrics( - connectionId: string, - routeRevision: number, - sequence: number, - metrics: Partial, -) { - const evidence = viewerQualityEvidence( - connectionId, - routeRevision, - sequence, - ); - return { - ...evidence, - metrics: { ...evidence.metrics, ...metrics }, - }; -} - -async function correlateParentEdgeQualityEvidence( - parent: TestClient, - proof: ParentEdgeQualityProof = { - kind: "remote-loss", - packetsSentDelta: 100, - remotePacketsLostDelta: 30, - }, -) { - const viewerEvidence = await parent.inbox.next("viewer-quality-evidence"); - parent.socket.send( - JSON.stringify({ - type: "parent-edge-quality-evidence", - viewerPeerId: viewerEvidence.viewerPeerId, - guard: viewerEvidence.guard, - viewerSequence: viewerEvidence.sequence, - proof, - }), - ); - return viewerEvidence; -} - -async function sendTestOffer( - source: TestClient, - target: TestClient, - targetPeerId: string, - connectionId: string, -) { - source.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId, - payload: { - kind: "description", - connectionId, - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - return target.inbox.next("signal"); -} - -async function startRelayRelativeFpsFixture( - prefix: string, - now: () => number, - options: { - alternateRelayCapacity?: 0 | 1 | 2; - issueSfuToken?: (peerId: string) => Promise; - onSfuTokenIssue?: () => void; - prepareTimeoutMs?: number; - hostFirstCandidate?: boolean; - hostShareGeneration?: string; - secondChild?: boolean; - selectedEdgeTurn?: boolean; - sfu?: boolean; - } = {}, -) { - const harness = options.sfu - ? await startSfuHarness({ - now, - prepareTimeoutMs: options.prepareTimeoutMs, - selectedEdgeTurn: options.selectedEdgeTurn, - tokenIssuer: { - issueToken: async ({ peerId }) => { - options.onSfuTokenIssue?.(); - return options.issueSfuToken?.(peerId) ?? `token-${peerId}`; - }, - }, - }) - : await startHarness({ peerAssistedMedia: true, now }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted( - await authenticate( - host, - harness.room, - "host", - `${prefix}-host`, - 1, - options.hostShareGeneration, - ), - ); - const parent = await openClient(harness.webSocketUrl); - const parentAuth = peerAssisted( - await authenticate( - parent, - harness.room, - "viewer", - `${prefix}-parent`, - options.secondChild || options.hostFirstCandidate ? 2 : 1, - ), - ); - const releasedHostSlot = options.hostFirstCandidate - ? await openClient(harness.webSocketUrl) - : undefined; - if (releasedHostSlot) { - await authenticate( - releasedHostSlot, - harness.room, - "viewer", - `${prefix}-released-host-slot`, - 0, - ); - } - const alternate = await openClient(harness.webSocketUrl); - const alternateAuth = peerAssisted( - await authenticate( - alternate, - harness.room, - "viewer", - `${prefix}-alternate`, - options.alternateRelayCapacity ?? 1, - ), - ); - const child = await openClient(harness.webSocketUrl); - const childAuth = peerAssisted( - await authenticate(child, harness.room, "viewer", `${prefix}-child`, 0), - ); - const secondChild = options.secondChild - ? await openClient(harness.webSocketUrl) - : undefined; - const secondChildAuth = secondChild - ? peerAssisted( - await authenticate( - secondChild, - harness.room, - "viewer", - `${prefix}-second-child`, - 0, - ), - ) - : undefined; - expect(parentAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: hostAuth.peerId, - }); - expect(alternateAuth.routeAssignment.upstream).toEqual({ kind: "peer", - peerId: options.hostFirstCandidate ? parentAuth.peerId : hostAuth.peerId }); - expect(childAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: parentAuth.peerId, - }); - expect(secondChildAuth?.routeAssignment.upstream).toEqual( - secondChildAuth - ? { kind: "peer", peerId: parentAuth.peerId } - : undefined, - ); - let routeRevision = secondChildAuth?.routeRevision ?? childAuth.routeRevision; - if (secondChildAuth) { - await nextActiveRouteRevision(child, routeRevision); - } - if (releasedHostSlot) { - releasedHostSlot.socket.close(); - const [releasedRoute] = await Promise.all([ - nextActiveRouteAfter(child, routeRevision), - nextActiveRouteAfter(host, routeRevision), - nextActiveRouteAfter(parent, routeRevision), - nextActiveRouteAfter(alternate, routeRevision), - ]); - routeRevision = releasedRoute.revision; - } - - const parentConnectionId = `${prefix}_parent_connection`; - await sendTestOffer(host, parent, parentAuth.peerId, parentConnectionId); - - const childConnectionId = `${prefix}_child_connection`; - await sendTestOffer(parent, child, childAuth.peerId, childConnectionId); - - const secondChildConnectionId = `${prefix}_second_child_connection`; - if (secondChild && secondChildAuth) { - await sendTestOffer( - parent, - secondChild, - secondChildAuth.peerId, - secondChildConnectionId, - ); - } - - return { - alternate, - alternateAuth, - child, - childAuth, - childConnectionId, - harness, - host, - hostAuth, - parent, - parentAuth, - parentConnectionId, - routeRevision, - secondChild, - secondChildAuth, - secondChildConnectionId, - }; -} - -async function correlateRelayParentInboundFps( - fixture: Awaited>, - framesPerSecond: number, - sequence = 0, -) { - fixture.parent.socket.send( - JSON.stringify( - viewerQualityEvidenceWithMetrics( - fixture.parentConnectionId, - fixture.routeRevision, - sequence, - { framesPerSecond }, - ), - ), - ); - while (true) { - const evidence = await fixture.host.inbox.next("viewer-quality-evidence"); - if (evidence.viewerPeerId !== fixture.parentAuth.peerId) { - continue; - } - fixture.host.socket.send( - JSON.stringify({ - type: "parent-edge-quality-evidence", - viewerPeerId: evidence.viewerPeerId, - guard: evidence.guard, - viewerSequence: evidence.sequence, - proof: { kind: "sending", packetsSentDelta: 1_500 }, - }), - ); - await new Promise((resolve) => setTimeout(resolve, 0)); - return; - } -} - -async function correlateRelayChildFps( - fixture: Awaited>, - sequence: number, - framesPerSecond: number, - child = fixture.child, - connectionId = fixture.childConnectionId, -) { - child.socket.send( - JSON.stringify( - viewerQualityEvidenceWithMetrics( - connectionId, - fixture.routeRevision, - sequence, - { framesPerSecond }, - ), - ), - ); - await correlateParentEdgeQualityEvidence(fixture.parent, { - kind: "sender-limited", - packetsSentDelta: 1_500, - reason: "bandwidth", - }); - await new Promise((resolve) => setTimeout(resolve, 0)); -} - -async function confirmRelayChildRelativeFps( - fixture: Awaited>, - child: TestClient, - connectionId: string, - advanceWindow: () => void, -) { - for (let sequence = 0; sequence < 3; sequence += 1) { - await correlateRelayChildFps( - fixture, - sequence, - 10, - child, - connectionId, - ); - if (sequence < 2) advanceWindow(); - } -} - -async function confirmRelayParentChildRelativeFps( - fixture: Awaited>, - childIndex: 0 | 1, - advanceWindow: () => void, -) { - const child = childIndex === 0 ? fixture.child : fixture.secondChild!; - const connectionId = childIndex === 0 - ? fixture.childConnectionId - : fixture.secondChildConnectionId; - await correlateRelayParentInboundFps(fixture, 30, childIndex); - await confirmRelayChildRelativeFps( - fixture, - child, - connectionId, - advanceWindow, - ); -} - -async function confirmRelayChildSevere( - fixture: Awaited>, - child: TestClient, - connectionId: string, - advanceWindow: () => void, -) { - for (let sequence = 0; sequence < 3; sequence += 1) { - child.socket.send(JSON.stringify(viewerQualityEvidenceWithMetrics( - connectionId, - fixture.routeRevision, - sequence, - { framesDecodedDelta: 0 }, - ))); - await correlateParentEdgeQualityEvidence(fixture.parent); - await new Promise((resolve) => setTimeout(resolve, 0)); - if (sequence < 2) advanceWindow(); - } -} - -async function prepareHostParentQualityProbe( - fixture: Awaited>, - advanceWindow: () => void, - connectionId: string, -) { - fixture.alternate.socket.close(); - const [current] = await Promise.all([ - nextActiveRouteAfter(fixture.child, fixture.routeRevision), - nextActiveRouteAfter(fixture.parent, fixture.routeRevision), - nextActiveRouteAfter(fixture.host, fixture.routeRevision), - ]); - fixture.routeRevision = current.revision; - await confirmRelayChildSevere( - fixture, - fixture.child, - fixture.childConnectionId, - advanceWindow, - ); - const [viewerPrepare, hostPrepare] = await Promise.all([ - nextPreparedRoute(fixture.child), - nextPreparedRoute(fixture.host), - ]); - expect(viewerPrepare.assignment.upstream).toEqual({ - kind: "peer", - peerId: fixture.hostAuth.peerId, - }); - expect(hostPrepare.revision).toBe(viewerPrepare.revision); - await sendTestOffer( - fixture.host, - fixture.child, - fixture.childAuth.peerId, - connectionId, - ); - return { viewerPrepare, hostPrepare }; -} - -async function reconnectRelayQualityEdges( - fixture: Awaited>, - suffix: string, -) { - const secondChild = fixture.secondChild!; - const secondChildAuth = fixture.secondChildAuth!; - fixture.parentConnectionId = `${suffix}_parent_connection`; - await sendTestOffer( - fixture.host, - fixture.parent, - fixture.parentAuth.peerId, - fixture.parentConnectionId, - ); - fixture.secondChildConnectionId = `${suffix}_second_child_connection`; - await sendTestOffer( - fixture.parent, - secondChild, - secondChildAuth.peerId, - fixture.secondChildConnectionId, - ); -} - -async function startSfuHarness(options: { - tokenIssuer: SfuTokenIssuer; - selectedEdgeTurn?: boolean; - persistent?: boolean; - prepareTimeoutMs?: number; - sfuIngressCapacity?: number; - sfuEgressCapacity?: number; - endpointMediaCopyCapacity?: number; - viewerDisconnectGraceMs?: number; - stunUrls?: readonly string[]; - now?: () => number; - roomControl?: FakeSfuRoomControl; - hostOfflineCheckMs?: number; - drainRetryMs?: number; - turnAllocationCapacity?: number; -}): Promise { - const roomStore = new RoomStore({ - ttlMs: 14_400_000, - maxRooms: 10, - maxViewersPerRoom: 8, - database: options.persistent ? new RoomDatabase(":memory:") : undefined, - }); - if (options.persistent) { - roomStore.createRoom(); - } - const room = roomStore.createRoom(); - const httpServer = createServer((_request, response) => { - response.statusCode = 404; - response.end(); - }); - const sfuAdmission = new SfuResourceAdmission({ - ingressCapacity: - options.sfuIngressCapacity ?? TEST_SFU_INGRESS_CAPACITY, - egressCapacity: - options.sfuEgressCapacity ?? TEST_SFU_EGRESS_CAPACITY, - }); - const turnAllocationCapacity = - options.turnAllocationCapacity ?? TEST_TURN_ALLOCATION_CAPACITY; - const turnAdmission = options.selectedEdgeTurn - ? new TurnAllocationAdmission({ capacity: turnAllocationCapacity }) - : undefined; - const sfuRoomControl = options.roomControl ?? new FakeSfuRoomControl(); - const signaling = new SignalingServer({ - server: httpServer, - roomStore, - peerAssistedMedia: true, - endpointMediaCopyCapacity: - options.endpointMediaCopyCapacity ?? 2, - sfuFallback: { - url: "wss://sfu.example.test", - tokenIssuer: options.tokenIssuer, - admission: sfuAdmission, - roomControl: sfuRoomControl, - prepareTimeoutMs: options.prepareTimeoutMs, - hostOfflineCheckMs: options.hostOfflineCheckMs, - drainRetryMs: options.drainRetryMs, - }, - ...(options.selectedEdgeTurn - ? { - selectedEdgeTurn: { - config: { - urls: ["turn:turn.example.test:3478?transport=udp"], - sharedSecret: "t".repeat(32), - credentialTtlSeconds: 120, - allocationCapacity: turnAllocationCapacity, - }, - admission: turnAdmission!, - }, - } - : {}), - ice: { - stunUrls: options.stunUrls ?? [], - }, - allowedOrigins: new Set([allowedOrigin]), - siteAccessAtUpgrade: () => true, - publicBaseUrl: new URL("https://share.example.test"), - now: options.now, - authenticationTimeoutMs: 500, - viewerDisconnectGraceMs: options.viewerDisconnectGraceMs ?? 50, - heartbeatIntervalMs: 60_000, - cleanupIntervalMs: 60_000, - }); - runningServer = { - httpServer, - roomStore, - listen: (port = 0, host = "127.0.0.1") => - new Promise((resolve, reject) => { - httpServer.once("error", reject); - httpServer.listen(port, host, () => { - httpServer.off("error", reject); - const address = httpServer.address(); - if (!address || typeof address === "string") { - reject(new Error("Test HTTP server has no TCP address")); - return; - } - resolve(address.port); - }); - }), - async close() { - await signaling.close(); - await new Promise((resolve, reject) => { - httpServer.close((error) => (error ? reject(error) : resolve())); - }); - roomStore.close(); - }, - }; - const port = await runningServer.listen(); - return { - baseUrl: `http://127.0.0.1:${port}`, - webSocketUrl: `ws://127.0.0.1:${port}/signal`, - roomStore, - room, - sfuAdmission, - turnAdmission, - sfuRoomControl, - }; -} - async function openClient( webSocketUrl: string, cookie?: string, @@ -906,18 +332,6 @@ function peerAssisted( return message; } -async function nextActiveRouteAfter( - client: TestClient, - revision: number, -) { - while (true) { - const message = await client.inbox.next("route-update"); - if (message.phase === "active" && message.revision > revision) { - return message; - } - } -} - async function nextViewerPresenceMatching( client: TestClient, predicate: ( @@ -954,44 +368,6 @@ async function nextPreparedRoute(client: TestClient) { } } -async function completePreparedPeerMigration( - viewer: TestClient, - newParent: TestClient, - viewerPeerId: string, - newParentPeerId: string, - previousRevision: number, - connectionId: string, -) { - const [viewerPrepare, parentPrepare] = await Promise.all([ - nextPreparedRoute(viewer), - nextPreparedRoute(newParent), - ]); - expect(viewerPrepare).toMatchObject({ - phase: "prepare", - assignment: { upstream: { kind: "peer", peerId: newParentPeerId } }, - }); - expect(parentPrepare.revision).toBe(viewerPrepare.revision); - await sendTestOffer(newParent, viewer, viewerPeerId, connectionId); - viewer.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: newParentPeerId, - payload: { - kind: "description", - connectionId, - description: { type: "answer", sdp: "v=0\r\n" }, - }, - })); - expect(await newParent.inbox.next("signal")).toMatchObject({ - fromPeerId: viewerPeerId, - }); - viewer.socket.send(JSON.stringify({ - type: "route-ready", - revision: viewerPrepare.revision, - phase: "prepare", - })); - return nextActiveRouteAfter(viewer, previousRevision); -} - async function nextActiveRouteRevision(client: TestClient, revision: number) { while (true) { const message = await client.inbox.next("route-update"); @@ -1001,630 +377,6 @@ async function nextActiveRouteRevision(client: TestClient, revision: number) { } } -async function prepareFallbackForTwoViewers( - webSocketUrl: string, - room: CreatedRoom, - prefix: string, - shareGeneration?: string, - options: { viewerParentSibling?: boolean } = {}, -) { - const host = await openClient(webSocketUrl); - const hostAuth = peerAssisted( - await authenticate( - host, - room, - "host", - `${prefix}-host-client`, - 1, - shareGeneration, - ), - ); - const failedViewer = await openClient(webSocketUrl); - const failedAuth = peerAssisted( - await authenticate( - failedViewer, - room, - "viewer", - `${prefix}-failed-client`, - options.viewerParentSibling ? 0 : 1, - ), - ); - const rootViewer = await openClient(webSocketUrl); - const rootAuth = peerAssisted( - await authenticate( - rootViewer, - room, - "viewer", - `${prefix}-root-client`, - options.viewerParentSibling ? 2 : 1, - ), - ); - let rootSibling: TestClient | undefined; - let rootSiblingAuth: - | ReturnType - | undefined; - if (options.viewerParentSibling) { - rootSibling = await openClient(webSocketUrl); - rootSiblingAuth = peerAssisted( - await authenticate( - rootSibling, - room, - "viewer", - `${prefix}-root-sibling-client`, - 0, - ), - ); - expect(rootSiblingAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: rootAuth.peerId, - }); - } - const directRoute = rootSiblingAuth - ? await nextActiveRouteRevision( - failedViewer, - rootSiblingAuth.routeRevision, - ) - : await nextActiveRouteAfter(failedViewer, failedAuth.routeRevision); - host.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: failedAuth.peerId, - payload: { - kind: "description", - connectionId: `${prefix}-direct-connection`, - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await failedViewer.inbox.next("signal"); - failedViewer.socket.send( - JSON.stringify({ - type: "route-failed", - revision: directRoute.revision, - phase: "active", - connectionId: `${prefix}-direct-connection`, - }), - ); - const reparented = await nextActiveRouteAfter( - failedViewer, - directRoute.revision, - ); - rootViewer.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: failedAuth.peerId, - payload: { - kind: "description", - connectionId: `${prefix}-deep-connection`, - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await failedViewer.inbox.next("signal"); - failedViewer.socket.send( - JSON.stringify({ - type: "route-failed", - revision: reparented.revision, - phase: "active", - connectionId: `${prefix}-deep-connection`, - }), - ); - const hostPrepare = await nextPreparedRoute(host); - const rootPrepare = await nextPreparedRoute(failedViewer); - await host.inbox.next("sfu-config"); - await failedViewer.inbox.next("sfu-config"); - return { - host, - hostAuth, - failedViewer, - failedAuth, - rootViewer, - rootAuth, - rootSibling, - rootSiblingAuth, - hostPrepare, - rootPrepare, - }; -} - -async function failSingleViewerDirect( - webSocketUrl: string, - room: CreatedRoom, - prefix: string, - hostPresence = false, -) { - const host = await openClient(webSocketUrl); - const hostAuth = peerAssisted( - await authenticate( - host, - room, - "host", - `${prefix}-host`, - 1, - `${prefix}-share`, - hostPresence ? { viewerPresence: true } : {}, - ), - ); - const viewer = await openClient(webSocketUrl); - const viewerAuth = peerAssisted( - await authenticate(viewer, room, "viewer", `${prefix}-viewer`), - ); - const oldConnectionId = `${prefix}-direct-connection`; - host.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: viewerAuth.peerId, - payload: { - kind: "description", - connectionId: oldConnectionId, - description: { type: "offer", sdp: "v=0\r\n" }, - }, - })); - await viewer.inbox.next("signal"); - viewer.socket.send(JSON.stringify({ - type: "route-failed", - revision: viewerAuth.routeRevision, - phase: "active", - connectionId: oldConnectionId, - })); - return { host, hostAuth, viewer, viewerAuth, oldConnectionId }; -} - -async function activateSingleViewerSfu( - webSocketUrl: string, - room: CreatedRoom, - prefix: string, - hostPresence = false, -) { - const direct = await failSingleViewerDirect( - webSocketUrl, - room, - prefix, - hostPresence, - ); - const { host, viewer } = direct; - const hostPrepare = await nextPreparedRoute(host); - const viewerPrepare = await nextPreparedRoute(viewer); - await Promise.all([host.inbox.next("sfu-config"), viewer.inbox.next("sfu-config")]); - for (const client of [host, viewer]) { - client.socket.send(JSON.stringify({ - type: "route-ready", - revision: viewerPrepare.revision, - phase: "prepare", - })); - } - await Promise.all([ - nextActiveRouteRevision(host, hostPrepare.revision), - nextActiveRouteRevision(viewer, viewerPrepare.revision), - ]); - return { ...direct, revision: viewerPrepare.revision }; -} - -async function prepareSelectedHostIngressReplacement( - harness: SignalHarness, - prefix: string, -) { - const active = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - prefix, - ); - active.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: active.revision, - phase: "active", - connectionId: null, - })); - const activeGrant = await active.host.inbox.next("selected-edge-turn"); - if (activeGrant.edgeKind !== "host-sfu-ingress") { - throw new Error("expected an active Host SFU ingress grant"); - } - active.host.socket.send(JSON.stringify({ - type: "route-ready", - revision: active.revision, - phase: "active", - })); - await vi.waitFor(() => - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 1 }), - ); - active.viewer.socket.send(JSON.stringify({ - type: "relay-capacity", - downstreamEdges: 0, - })); - await new Promise((resolve) => setTimeout(resolve, 0)); - - const candidateViewer = await openClient(harness.webSocketUrl); - const candidateAuth = peerAssisted(await authenticate( - candidateViewer, - harness.room, - "viewer", - `${prefix}-candidate-viewer`, - 0, - )); - expect(candidateAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: active.hostAuth.peerId, - }); - const candidateConnectionId = `${prefix}-candidate-connection`; - await sendTestOffer( - active.host, - candidateViewer, - candidateAuth.peerId, - candidateConnectionId, - ); - candidateViewer.socket.send(JSON.stringify({ - type: "route-failed", - revision: candidateAuth.routeRevision, - phase: "active", - connectionId: candidateConnectionId, - })); - const [hostPrepare, activeViewerPrepare, candidatePrepare] = - await Promise.all([ - nextPreparedRoute(active.host), - nextPreparedRoute(active.viewer), - nextPreparedRoute(candidateViewer), - ]); - expect(activeViewerPrepare.revision).toBe(hostPrepare.revision); - expect(candidatePrepare.revision).toBe(hostPrepare.revision); - await Promise.all([ - active.host.inbox.next("sfu-config"), - active.viewer.inbox.next("sfu-config"), - candidateViewer.inbox.next("sfu-config"), - ]); - - active.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: hostPrepare.revision, - phase: "prepare", - connectionId: null, - })); - const candidateGrant = await active.host.inbox.next("selected-edge-turn"); - if (candidateGrant.edgeKind !== "host-sfu-ingress") { - throw new Error("expected a pending Host SFU ingress grant"); - } - expect(candidateGrant.newConnectionId).not.toBe(activeGrant.newConnectionId); - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 2 }); - return { - active, - activeGrant, - candidateGrant, - candidateViewer, - hostPrepare, - }; -} - -async function prepareSfuHostParentQualityProbe( - harness: SignalHarness, - active: Awaited>, - prefix: string, - advanceWindow: () => void, -) { - active.viewer.socket.send(JSON.stringify({ - type: "relay-capacity", - downstreamEdges: 0, - })); - const parent = await openClient(harness.webSocketUrl); - const parentAuth = peerAssisted(await authenticate( - parent, - harness.room, - "viewer", - `${prefix}-parent`, - 1, - )); - expect(parentAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: active.hostAuth.peerId, - }); - const hostFiller = await openClient(harness.webSocketUrl); - await authenticate( - hostFiller, - harness.room, - "viewer", - `${prefix}-filler`, - 0, - ); - const child = await openClient(harness.webSocketUrl); - const childAuth = peerAssisted(await authenticate( - child, - harness.room, - "viewer", - `${prefix}-child`, - 0, - )); - expect(childAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: parentAuth.peerId, - }); - hostFiller.socket.close(); - const current = await nextActiveRouteAfter(child, childAuth.routeRevision); - await Promise.all([ - nextActiveRouteRevision(active.host, current.revision), - nextActiveRouteRevision(active.viewer, current.revision), - nextActiveRouteRevision(parent, current.revision), - ]); - - await sendTestOffer( - active.host, - parent, - parentAuth.peerId, - `${prefix}_parent_connection`, - ); - const childConnectionId = `${prefix}_child_connection`; - await sendTestOffer(parent, child, childAuth.peerId, childConnectionId); - for (let sequence = 0; sequence < 3; sequence += 1) { - child.socket.send(JSON.stringify(viewerQualityEvidenceWithMetrics( - childConnectionId, - current.revision, - sequence, - { framesDecodedDelta: 0 }, - ))); - await correlateParentEdgeQualityEvidence(parent); - advanceWindow(); - } - const [childPrepare, hostPrepare] = await Promise.all([ - nextPreparedRoute(child), - nextPreparedRoute(active.host), - ]); - expect(childPrepare.assignment.upstream).toEqual({ - kind: "peer", - peerId: active.hostAuth.peerId, - }); - expect(hostPrepare.revision).toBe(childPrepare.revision); - return { - child, - childPrepare, - current, - hostPrepare, - parentAuth, - }; -} - -async function prepareTwoSelectedPeerEdgeCandidates( - webSocketUrl: string, - room: CreatedRoom, - prefix: string, -) { - const host = await openClient(webSocketUrl); - const hostAuth = peerAssisted(await authenticate( - host, - room, - "host", - `${prefix}-host`, - 1, - `${prefix}-share`, - )); - const firstParent = await openClient(webSocketUrl); - const firstParentAuth = peerAssisted(await authenticate( - firstParent, - room, - "viewer", - `${prefix}-first-parent`, - 1, - )); - const secondParent = await openClient(webSocketUrl); - const secondParentAuth = peerAssisted(await authenticate( - secondParent, - room, - "viewer", - `${prefix}-second-parent`, - 1, - )); - await new Promise((resolve) => setTimeout(resolve, 0)); - - const firstViewer = await openClient(webSocketUrl); - const firstAuth = peerAssisted(await authenticate( - firstViewer, - room, - "viewer", - `${prefix}-first-viewer`, - 0, - )); - const secondViewer = await openClient(webSocketUrl); - const secondAuth = peerAssisted(await authenticate( - secondViewer, - room, - "viewer", - `${prefix}-second-viewer`, - 0, - )); - expect(firstAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: firstParentAuth.peerId, - }); - expect(secondAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: secondParentAuth.peerId, - }); - const firstCurrent = await nextActiveRouteAfter( - firstViewer, - firstAuth.routeRevision, - ); - expect(firstCurrent.revision).toBe(secondAuth.routeRevision); - - const firstConnectionId = `${prefix}-first-edge`; - const secondConnectionId = `${prefix}-second-edge`; - await sendTestOffer( - firstParent, - firstViewer, - firstAuth.peerId, - firstConnectionId, - ); - await sendTestOffer( - secondParent, - secondViewer, - secondAuth.peerId, - secondConnectionId, - ); - return { - host, - hostAuth, - firstParent, - firstParentAuth, - firstViewer, - firstAuth, - firstConnectionId, - secondParent, - secondParentAuth, - secondViewer, - secondAuth, - secondConnectionId, - revision: firstCurrent.revision, - }; -} - -function failPeerEdgeCandidate( - viewer: TestClient, - revision: number, - connectionId: string, -): void { - viewer.socket.send(JSON.stringify({ - type: "route-failed", - revision, - phase: "active", - connectionId, - })); -} - -async function startPeerSelectedCandidate( - viewer: TestClient, - parent: TestClient, - revision: number, - connectionId: string, -) { - failPeerEdgeCandidate(viewer, revision, connectionId); - const viewerGrant = await viewer.inbox.next("selected-edge-turn"); - if (viewerGrant.edgeKind !== "peer-selected") { - throw new Error("expected a peer-selected grant"); - } - let parentGrant: Extract; - do { - parentGrant = await parent.inbox.next("selected-edge-turn"); - } while (parentGrant.newConnectionId !== viewerGrant.newConnectionId); - expect(parentGrant).toEqual(viewerGrant); - return viewerGrant; -} - -async function failActiveSfuRoot(client: TestClient, revision: number) { - client.socket.send(JSON.stringify({ - type: "refresh-sfu", - revision, - })); - await client.inbox.next("sfu-config"); - client.socket.send(JSON.stringify({ - type: "route-failed", - revision, - phase: "active", - connectionId: null, - })); -} - -function requestHealthyReselection(client: TestClient, revision: number): void { - client.socket.send(JSON.stringify({ - type: "sfu-reselection-ready", - revision, - })); -} - -async function startPeerSelectedTurn( - viewer: TestClient, - parent: TestClient, - revision: number, -) { - await failActiveSfuRoot(viewer, revision); - const [viewerGrant, parentGrant] = await Promise.all([ - viewer.inbox.next("selected-edge-turn"), - parent.inbox.next("selected-edge-turn"), - ]); - expect(parentGrant).toEqual(viewerGrant); - if (viewerGrant.edgeKind !== "peer-selected") { - throw new Error("expected a peer-selected grant"); - } - return viewerGrant; -} - -async function exhaustDeepViewerPeerRoutes( - webSocketUrl: string, - room: CreatedRoom, - prefix: string, -) { - const host = await openClient(webSocketUrl); - const hostAuth = peerAssisted( - await authenticate(host, room, "host", `${prefix}-host`), - ); - const firstRoot = await openClient(webSocketUrl); - const firstRootAuth = peerAssisted( - await authenticate(firstRoot, room, "viewer", `${prefix}-root-a`), - ); - const secondRoot = await openClient(webSocketUrl); - const secondRootAuth = peerAssisted( - await authenticate(secondRoot, room, "viewer", `${prefix}-root-b`), - ); - const failedViewer = await openClient(webSocketUrl); - const failedAuth = peerAssisted( - await authenticate(failedViewer, room, "viewer", `${prefix}-failed`), - ); - - firstRoot.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: failedAuth.peerId, - payload: { - kind: "description", - connectionId: `${prefix}-edge-a`, - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await failedViewer.inbox.next("signal"); - failedViewer.socket.send( - JSON.stringify({ - type: "route-failed", - revision: failedAuth.routeRevision, - phase: "active", - connectionId: `${prefix}-edge-a`, - }), - ); - const reparented = await nextActiveRouteAfter( - failedViewer, - failedAuth.routeRevision, - ); - expect(reparented.assignment.upstream).toEqual({ - kind: "peer", - peerId: secondRootAuth.peerId, - }); - - secondRoot.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: failedAuth.peerId, - payload: { - kind: "description", - connectionId: `${prefix}-edge-b`, - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await failedViewer.inbox.next("signal"); - failedViewer.socket.send( - JSON.stringify({ - type: "route-failed", - revision: reparented.revision, - phase: "active", - connectionId: `${prefix}-edge-b`, - }), - ); - - return { - host, - hostAuth, - firstRoot, - firstRootAuth, - secondRoot, - secondRootAuth, - failedViewer, - failedAuth, - }; -} - async function closeClient(client: TestClient): Promise { if (client.socket.readyState === WebSocket.CLOSED) { return; @@ -1634,14 +386,6 @@ async function closeClient(client: TestClient): Promise { await closed; } -function deferredVoid(): { promise: Promise; resolve: () => void } { - let resolve!: () => void; - const promise = new Promise((settle) => { - resolve = settle; - }); - return { promise, resolve }; -} - describe("WebSocket signaling", () => { it("answers only opted-in current sockets and rate-limits each socket", async () => { let now = Date.now(); @@ -1819,15 +563,32 @@ describe("WebSocket signaling", () => { for (const [index, displayName] of viewerNames.entries()) { const viewer = await openClient(harness.webSocketUrl); viewers.push(viewer); - await authenticate( - viewer, - harness.room, - "viewer", - `presence-viewer-${index}`, - 1, - undefined, - displayName ? { displayName } : {}, + const authenticated = peerAssisted( + await authenticate( + viewer, + harness.room, + "viewer", + `presence-viewer-${index}`, + 1, + undefined, + displayName ? { displayName } : {}, + ), ); + const prepared = await Promise.race([ + nextPreparedRoute(viewer), + host.inbox.next("error").then((error) => { + throw new Error(`route error: ${error.message}`); + }), + ]); + viewer.socket.send( + JSON.stringify({ + type: "route-ready", + revision: prepared.revision, + phase: "prepare", + }), + ); + expect(prepared.revision).toBeGreaterThan(authenticated.routeRevision); + await nextActiveRouteRevision(viewer, prepared.revision); } const full = await nextViewerPresenceMatching( @@ -2409,165 +1170,6 @@ describe("WebSocket signaling", () => { }); }); - it("strongly rotates and revokes every Viewer generation and media edge", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { - async issueToken(request) { - return `token-${request.peerId}`; - }, - }, - }); - const peers = await exhaustDeepViewerPeerRoutes( - harness.webSocketUrl, - harness.room, - "access-generation", - ); - const hostPrepare = await nextPreparedRoute(peers.host); - const branchPrepare = await nextPreparedRoute(peers.secondRoot); - const failedPrepare = await nextPreparedRoute(peers.failedViewer); - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 1, egress: 2 }); - await peers.host.inbox.next("sfu-config"); - await peers.secondRoot.inbox.next("sfu-config"); - await peers.failedViewer.inbox.next("sfu-config"); - for (const [client, revision] of [ - [peers.host, hostPrepare.revision], - [peers.secondRoot, branchPrepare.revision], - [peers.failedViewer, failedPrepare.revision], - ] as const) { - client.socket.send( - JSON.stringify({ type: "route-ready", revision, phase: "prepare" }), - ); - } - const hostActive = await nextActiveRouteAfter( - peers.host, - hostPrepare.revision - 1, - ); - expect(hostActive.assignment.sfuPublicationGeneration).toBeTruthy(); - - const revokedViewers = [ - peers.firstRoot, - peers.secondRoot, - peers.failedViewer, - ]; - const closeCodes = revokedViewers.map( - (viewer) => - new Promise((resolve) => - viewer.socket.once("close", (code) => resolve(code)), - ), - ); - for (const viewer of revokedViewers) { - viewer.socket.on("message", (data) => { - if ( - decodeServerMessage(data.toString()).type === "viewer-access-revoked" && - viewer.socket.readyState === WebSocket.OPEN - ) { - viewer.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: peers.hostAuth.peerId, - payload: { - kind: "candidate", - connectionId: "stale-revoked-generation", - candidate: null, - }, - }), - ); - } - }); - } - - peers.host.socket.send( - JSON.stringify({ type: "set-viewer-access", action: "rotate" }), - ); - for (const viewer of revokedViewers) { - expect(await viewer.inbox.next("viewer-access-revoked")).toEqual({ - type: "viewer-access-revoked", - viewerAuthorizationGeneration: - peers.firstRootAuth.viewerAuthorizationGeneration, - }); - } - expect(await Promise.all(closeCodes)).toEqual([4004, 4004, 4004]); - expect(harness.roomStore.getConnectedViewers(harness.room.roomId)).toEqual( - [], - ); - - let finalHostRoute = hostActive; - for (let change = 0; change < 6; change += 1) { - if ( - finalHostRoute.assignment.childPeerIds.length === 0 && - finalHostRoute.assignment.sfuPublicationGeneration === null - ) { - break; - } - finalHostRoute = await nextActiveRouteAfter( - peers.host, - finalHostRoute.revision, - ); - } - expect(finalHostRoute.assignment).toMatchObject({ - childPeerIds: [], - sfuPublicationGeneration: null, - }); - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 0, egress: 0 }); - await expect(peers.host.inbox.next("signal", 30)).rejects.toThrow( - "Timed out", - ); - - const rotated = await peers.host.inbox.next("viewer-access-updated"); - expect(rotated.viewerPolicy).toBe("private-link"); - expect(rotated.viewerAuthorizationGeneration).not.toBe( - peers.firstRootAuth.viewerAuthorizationGeneration, - ); - const inviteUrl = new URL(rotated.inviteUrl!); - const rotatedGrant = new URLSearchParams(inviteUrl.hash.slice(1)).get("v"); - expect(rotatedGrant).toBeTruthy(); - - const oldGrantViewer = await openClient(harness.webSocketUrl); - oldGrantViewer.socket.send( - JSON.stringify({ - type: "authenticate", - protocol: SIGNALING_PROTOCOL, - roomId: harness.room.roomId, - role: "viewer", - viewerGrant: harness.room.viewerGrant, - clientId: "old-generation-viewer", - }), - ); - expect(await oldGrantViewer.inbox.next("error")).toMatchObject({ - code: "INVALID_TOKEN", - }); - - const rotatedRoom = { - ...harness.room, - viewerGrant: rotatedGrant, - } satisfies CreatedRoom; - const currentViewer = await openClient(harness.webSocketUrl); - const currentAuth = await authenticate( - currentViewer, - rotatedRoom, - "viewer", - "current-generation-viewer", - ); - expect(currentAuth.viewerAuthorizationGeneration).toBe( - rotated.viewerAuthorizationGeneration, - ); - - const currentClosed = new Promise((resolve) => - currentViewer.socket.once("close", (code) => resolve(code)), - ); - peers.host.socket.send( - JSON.stringify({ type: "set-viewer-access", action: "revoke" }), - ); - expect(await currentViewer.inbox.next("viewer-access-revoked")).toMatchObject({ - viewerAuthorizationGeneration: rotated.viewerAuthorizationGeneration, - }); - expect(await currentClosed).toBe(4004); - expect(await peers.host.inbox.next("viewer-access-updated")).toMatchObject({ - viewerPolicy: "private-link", - inviteUrl: null, - }); - }); - it("lets viewers arrive first and replays their snapshot when the host joins", async () => { const harness = await startHarness(); const viewer = await openClient(harness.webSocketUrl); @@ -2708,7171 +1310,6 @@ describe("WebSocket signaling", () => { }); }); - it("advances a provisional peer route when the host arrives after viewers", async () => { - const harness = await startHarness({ peerAssistedMedia: true }); - const viewer = await openClient(harness.webSocketUrl); - const viewerAuth = peerAssisted( - await authenticate( - viewer, - harness.room, - "viewer", - "early-relay-viewer", - ), - ); - expect(viewerAuth).toMatchObject({ - routeRevision: 0, - routeAssignment: { upstream: { kind: "none" } }, - }); - - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted( - await authenticate(host, harness.room, "host", "early-relay-host"), - ); - expect(hostAuth.routeRevision).toBeGreaterThan(viewerAuth.routeRevision); - expect(await nextActiveRouteAfter(viewer, viewerAuth.routeRevision)).toMatchObject({ - revision: hostAuth.routeRevision, - assignment: { - upstream: { kind: "peer", peerId: hostAuth.peerId }, - }, - }); - }); - - it("authenticates a persistent room without a room expiry", async () => { - const harness = await startHarness({ persistent: true }); - expect(harness.room).toMatchObject({ roomId: "1", expiresAt: null }); - - const viewer = await openClient(harness.webSocketUrl); - const authenticated = await authenticate( - viewer, - harness.room, - "viewer", - "viewer-persistent-room", - ); - expect(authenticated.roomExpiresAt).toBeNull(); - expect(authenticated.hostOnline).toBe(false); - }); - - it("synchronizes strict last-wins quality settings across a peer-assisted room", async () => { - const harness = await startHarness({ peerAssistedMedia: true }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted( - await authenticate(host, harness.room, "host", "quality-host"), - ); - expect(hostAuth.qualitySettings).toEqual(defaultQualitySettings); - - const viewer = await openClient(harness.webSocketUrl); - const viewerAuth = peerAssisted( - await authenticate(viewer, harness.room, "viewer", "quality-viewer"), - ); - expect(viewerAuth.qualitySettings).toEqual(defaultQualitySettings); - await host.inbox.next("media-assignment"); - - host.socket.send( - JSON.stringify({ - type: "set-quality-settings", - qualitySettings: balancedQualitySettings, - }), - ); - expect(await viewer.inbox.next("quality-settings")).toEqual({ - type: "quality-settings", - qualitySettings: balancedQualitySettings, - }); - host.socket.send( - JSON.stringify({ - type: "set-quality-settings", - qualitySettings: lowQualitySettings, - }), - ); - expect(await viewer.inbox.next("quality-settings")).toEqual({ - type: "quality-settings", - qualitySettings: lowQualitySettings, - }); - - const lateViewer = await openClient(harness.webSocketUrl); - const lateViewerAuth = peerAssisted( - await authenticate( - lateViewer, - harness.room, - "viewer", - "quality-viewer-late", - ), - ); - expect(lateViewerAuth.qualitySettings).toEqual(lowQualitySettings); - await host.inbox.next("media-assignment"); - - viewer.socket.send( - JSON.stringify({ - type: "set-quality-settings", - qualitySettings: defaultQualitySettings, - }), - ); - expect((await viewer.inbox.next("error")).code).toBe("FORBIDDEN"); - await lateViewer.inbox.expectNone(30); - }); - - it("uses only explicitly advertised viewer relay slots without moving healthy edges", async () => { - const harness = await startHarness({ peerAssistedMedia: true }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted( - await authenticate(host, harness.room, "host", "capacity-host"), - ); - const first = await openClient(harness.webSocketUrl); - const firstAuth = peerAssisted( - await authenticate( - first, - harness.room, - "viewer", - "capacity-first", - null, - ), - ); - const second = await openClient(harness.webSocketUrl); - const secondAuth = peerAssisted( - await authenticate( - second, - harness.room, - "viewer", - "capacity-second", - null, - ), - ); - const pending = await openClient(harness.webSocketUrl); - const pendingAuth = peerAssisted( - await authenticate( - pending, - harness.room, - "viewer", - "capacity-pending", - null, - ), - ); - - expect(firstAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: hostAuth.peerId, - }); - expect("sfuStandbyUrl" in firstAuth).toBe(false); - expect(secondAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: hostAuth.peerId, - }); - expect(pendingAuth.routeAssignment.upstream).toEqual({ kind: "none" }); - - first.socket.send( - JSON.stringify({ type: "relay-capacity", downstreamEdges: 1 }), - ); - const pendingRoute = await nextActiveRouteAfter( - pending, - pendingAuth.routeRevision, - ); - expect(pendingRoute.assignment.upstream).toEqual({ - kind: "peer", - peerId: firstAuth.peerId, - }); - const secondRoute = await nextActiveRouteAfter( - second, - secondAuth.routeRevision, - ); - expect(secondRoute.assignment.upstream).toEqual({ - kind: "peer", - peerId: hostAuth.peerId, - }); - - host.socket.send( - JSON.stringify({ type: "relay-capacity", downstreamEdges: 1 }), - ); - expect((await host.inbox.next("error")).code).toBe("FORBIDDEN"); - }); - - it("applies capacity three to Viewer authentication and reconciliation", async () => { - const harness = await startHarness({ - peerAssistedMedia: true, - endpointMediaCopyCapacity: 3, - }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted( - await authenticate(host, harness.room, "host", "release-cap-host"), - ); - const relay = await openClient(harness.webSocketUrl); - const relayAuth = peerAssisted( - await authenticate( - relay, - harness.room, - "viewer", - "release-cap-relay", - 3, - ), - ); - const sibling = await openClient(harness.webSocketUrl); - const siblingAuth = peerAssisted( - await authenticate( - sibling, - harness.room, - "viewer", - "release-cap-sibling", - 2, - ), - ); - const child = await openClient(harness.webSocketUrl); - const childAuth = peerAssisted( - await authenticate( - child, - harness.room, - "viewer", - "release-cap-child", - 0, - ), - ); - const siblingChild = await openClient(harness.webSocketUrl); - const siblingChildAuth = peerAssisted( - await authenticate( - siblingChild, - harness.room, - "viewer", - "release-cap-sibling-child", - 0, - ), - ); - const pending = await openClient(harness.webSocketUrl); - const pendingAuth = peerAssisted( - await authenticate( - pending, - harness.room, - "viewer", - "release-cap-pending", - 0, - ), - ); - - expect(relayAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: hostAuth.peerId, - }); - expect(siblingAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: hostAuth.peerId, - }); - expect(childAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: hostAuth.peerId, - }); - expect(siblingChildAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: relayAuth.peerId, - }); - expect(pendingAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: relayAuth.peerId, - }); - - const replacementRelay = await openClient(harness.webSocketUrl); - const replacementAuth = peerAssisted( - await authenticate( - replacementRelay, - harness.room, - "viewer", - "release-cap-relay", - null, - ), - ); - expect(replacementAuth.peerId).toBe(relayAuth.peerId); - expect(replacementAuth.routeAssignment).toMatchObject({ - upstream: { kind: "peer", peerId: hostAuth.peerId }, - childPeerIds: [siblingChildAuth.peerId, pendingAuth.peerId], - }); - - replacementRelay.socket.send( - JSON.stringify({ type: "relay-capacity", downstreamEdges: 3 }), - ); - await pending.inbox.expectNone(40); - await replacementRelay.inbox.expectNone(40); - }); - - it("rescues an unassigned relay in one bounded route generation", async () => { - const harness = await startHarness({ peerAssistedMedia: true }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted( - await authenticate(host, harness.room, "host", "rescue-host"), - ); - const first = await openClient(harness.webSocketUrl); - const firstAuth = peerAssisted( - await authenticate( - first, - harness.room, - "viewer", - "rescue-first", - null, - ), - ); - const second = await openClient(harness.webSocketUrl); - const secondAuth = peerAssisted( - await authenticate( - second, - harness.room, - "viewer", - "rescue-second", - null, - ), - ); - const relay = await openClient(harness.webSocketUrl); - const relayAuth = peerAssisted( - await authenticate( - relay, - harness.room, - "viewer", - "rescue-relay", - null, - ), - ); - expect(relayAuth.routeAssignment.upstream).toEqual({ kind: "none" }); - - host.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: firstAuth.peerId, - payload: { - kind: "description", - connectionId: "admission-rescue-old-generation", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await first.inbox.next("signal"); - - relay.socket.send( - JSON.stringify({ type: "relay-capacity", downstreamEdges: 1 }), - ); - const [hostRoute, firstRoute, secondRoute, relayRoute] = await Promise.all([ - nextActiveRouteAfter(host, relayAuth.routeRevision), - nextActiveRouteAfter(first, relayAuth.routeRevision), - nextActiveRouteAfter(second, relayAuth.routeRevision), - nextActiveRouteAfter(relay, relayAuth.routeRevision), - ]); - - expect( - new Set([ - hostRoute.revision, - firstRoute.revision, - secondRoute.revision, - relayRoute.revision, - ]), - ).toEqual(new Set([relayAuth.routeRevision + 1])); - expect(hostRoute.assignment).toMatchObject({ - upstream: { kind: "none" }, - childPeerIds: [secondAuth.peerId, relayAuth.peerId], - sfuPublicationGeneration: null, - }); - expect(firstRoute.assignment).toMatchObject({ - upstream: { kind: "peer", peerId: relayAuth.peerId }, - childPeerIds: [], - }); - expect(secondRoute.assignment.upstream).toEqual({ - kind: "peer", - peerId: hostAuth.peerId, - }); - expect(relayRoute.assignment).toMatchObject({ - upstream: { kind: "peer", peerId: hostAuth.peerId }, - childPeerIds: [firstAuth.peerId], - }); - expect(hostRoute.assignment.childPeerIds).toHaveLength(2); - expect(relayRoute.assignment.childPeerIds).toHaveLength(1); - - first.inbox.ignore("route-update"); - first.inbox.ignore("media-assignment"); - first.socket.send( - JSON.stringify({ - type: "route-failed", - revision: firstRoute.revision, - phase: "active", - connectionId: "admission-rescue-old-generation", - }), - ); - await first.inbox.expectNone(30); - }); - - it("keeps the P2P authenticated wire unchanged and forbids profile updates", async () => { - const harness = await startHarness(); - const host = await openClient(harness.webSocketUrl); - const hostAuth = await authenticate( - host, - harness.room, - "host", - "p2p-quality-host", - ); - expect("mediaMode" in hostAuth).toBe(false); - expect("qualitySettings" in hostAuth).toBe(false); - expect("sfuStandbyUrl" in hostAuth).toBe(false); - - const viewer = await openClient(harness.webSocketUrl); - const viewerAuth = await authenticate( - viewer, - harness.room, - "viewer", - "p2p-quality-viewer", - ); - expect("mediaMode" in viewerAuth).toBe(false); - expect("qualitySettings" in viewerAuth).toBe(false); - expect("sfuStandbyUrl" in viewerAuth).toBe(false); - await host.inbox.next("peer-joined"); - - host.socket.send( - JSON.stringify({ - type: "set-quality-settings", - qualitySettings: balancedQualitySettings, - }), - ); - expect((await host.inbox.next("error")).code).toBe("FORBIDDEN"); - viewer.socket.send( - JSON.stringify({ - type: "set-quality-settings", - qualitySettings: lowQualitySettings, - }), - ); - expect((await viewer.inbox.next("error")).code).toBe("FORBIDDEN"); - viewer.socket.send( - JSON.stringify({ type: "relay-capacity", downstreamEdges: 1 }), - ); - expect((await viewer.inbox.next("error")).code).toBe("FORBIDDEN"); - await viewer.inbox.expectNone(30); - }); - - it("enables hybrid routing for every room while ordinary ICE stays STUN-only", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { issueToken: async () => "unused-test-token" }, - stunUrls: ["stun:stun.example.test:3478"], - }); - const ordinaryRoom = harness.roomStore.createRoom(); - - const hybridHost = await openClient(harness.webSocketUrl); - const hybridHostAuth = peerAssisted( - await authenticate( - hybridHost, - harness.room, - "host", - "allowlisted-host", - ), - ); - expect(hybridHostAuth.sfuStandbyUrl).toBe("wss://sfu.example.test"); - expect(hybridHostAuth.iceConfig).toEqual({ - iceServers: [{ urls: ["stun:stun.example.test:3478"] }], - }); - - const ordinaryHost = await openClient(harness.webSocketUrl); - const ordinaryHostAuth = peerAssisted( - await authenticate( - ordinaryHost, - ordinaryRoom, - "host", - "ordinary-host", - ), - ); - expect(ordinaryHostAuth.mediaMode).toBe("peer-assisted"); - expect(ordinaryHostAuth.sfuStandbyUrl).toBe("wss://sfu.example.test"); - expect(ordinaryHostAuth.iceConfig).toEqual(hybridHostAuth.iceConfig); - expect( - ordinaryHostAuth.iceConfig.iceServers.every( - (server) => !("username" in server) && !("credential" in server), - ), - ).toBe(true); - - const ordinaryViewer = await openClient(harness.webSocketUrl); - const ordinaryViewerAuth = peerAssisted( - await authenticate( - ordinaryViewer, - ordinaryRoom, - "viewer", - "ordinary-viewer", - ), - ); - expect(ordinaryViewerAuth.mediaMode).toBe("peer-assisted"); - expect(ordinaryViewerAuth.sfuStandbyUrl).toBe("wss://sfu.example.test"); - expect(ordinaryViewerAuth.iceConfig).toEqual(hybridHostAuth.iceConfig); - - await closeClient(hybridHost); - await closeClient(ordinaryHost); - await closeClient(ordinaryViewer); - }); - - it("keeps all-room hybrid lifecycles independent", async () => { - const harness = await startHarness({ - peerAssistedMedia: true, - viewerDisconnectGraceMs: 500, - }); - const ordinaryRoom = harness.roomStore.createRoom(); - - const hybridHost = await openClient(harness.webSocketUrl); - const hybridHostAuth = peerAssisted( - await authenticate(hybridHost, harness.room, "host", "lifecycle-hybrid-host"), - ); - const hybridViewer = await openClient(harness.webSocketUrl); - const hybridViewerAuth = peerAssisted( - await authenticate( - hybridViewer, - harness.room, - "viewer", - "lifecycle-hybrid-viewer", - ), - ); - await hybridHost.inbox.next("media-assignment"); - hybridHost.socket.send( - JSON.stringify({ - type: "set-quality-settings", - qualitySettings: lowQualitySettings, - }), - ); - await hybridViewer.inbox.next("quality-settings"); - - const ordinaryHost = await openClient(harness.webSocketUrl); - await authenticate( - ordinaryHost, - ordinaryRoom, - "host", - "lifecycle-ordinary-host", - ); - const ordinaryViewer = await openClient(harness.webSocketUrl); - const ordinaryViewerAuth = peerAssisted( - await authenticate( - ordinaryViewer, - ordinaryRoom, - "viewer", - "lifecycle-ordinary-viewer", - ), - ); - await ordinaryHost.inbox.next("media-assignment"); - - const hybridHostClosed = new Promise((resolve) => - hybridHost.socket.once("close", (code) => resolve(code)), - ); - const ordinaryHostClosed = new Promise((resolve) => - ordinaryHost.socket.once("close", (code) => resolve(code)), - ); - hybridHost.socket.send(JSON.stringify({ type: "stop-sharing" })); - ordinaryHost.socket.send(JSON.stringify({ type: "stop-sharing" })); - await hybridViewer.inbox.next("sharing-stopped"); - await hybridViewer.inbox.next("host-status"); - await ordinaryViewer.inbox.next("sharing-stopped"); - await ordinaryViewer.inbox.next("host-status"); - expect(await hybridHostClosed).toBe(1000); - expect(await ordinaryHostClosed).toBe(1000); - - await closeClient(hybridViewer); - await closeClient(ordinaryViewer); - const resumedHybridViewer = await openClient(harness.webSocketUrl); - const resumedHybridViewerAuth = peerAssisted( - await authenticate( - resumedHybridViewer, - harness.room, - "viewer", - "lifecycle-hybrid-viewer", - ), - ); - expect(resumedHybridViewerAuth).toMatchObject({ - peerId: hybridViewerAuth.peerId, - connectionId: null, - qualitySettings: lowQualitySettings, - }); - const resumedOrdinaryViewer = await openClient(harness.webSocketUrl); - const resumedOrdinaryViewerAuth = peerAssisted( - await authenticate( - resumedOrdinaryViewer, - ordinaryRoom, - "viewer", - "lifecycle-ordinary-viewer", - ), - ); - expect(resumedOrdinaryViewerAuth.peerId).toBe(ordinaryViewerAuth.peerId); - expect(resumedOrdinaryViewerAuth.connectionId).toBeNull(); - expect(resumedOrdinaryViewerAuth.mediaMode).toBe("peer-assisted"); - expect(resumedOrdinaryViewerAuth.qualitySettings).toEqual( - defaultQualitySettings, - ); - - const resumedHybridHost = await openClient(harness.webSocketUrl); - const resumedHybridHostAuth = peerAssisted( - await authenticate( - resumedHybridHost, - harness.room, - "host", - "lifecycle-hybrid-host", - ), - ); - expect(resumedHybridHostAuth).toMatchObject({ - peerId: hybridHostAuth.peerId, - qualitySettings: lowQualitySettings, - }); - const resumedOrdinaryHost = await openClient(harness.webSocketUrl); - await authenticate( - resumedOrdinaryHost, - ordinaryRoom, - "host", - "lifecycle-ordinary-host", - ); - resumedHybridHost.socket.send(JSON.stringify({ type: "abandon-room" })); - await resumedHybridHost.inbox.next("room-closed"); - await resumedHybridViewer.inbox.next("room-closed"); - expect(harness.roomStore.size).toBe(1); - - resumedOrdinaryHost.socket.send(JSON.stringify({ type: "abandon-room" })); - await resumedOrdinaryHost.inbox.next("room-closed"); - await resumedOrdinaryViewer.inbox.next("room-closed"); - expect(harness.roomStore.size).toBe(0); - }); - - it("forwards only current, bounded ordinary-P2P viewer C evidence", async () => { - let now = Date.now(); - const harness = await startHarness({ now: () => now }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = await authenticate( - host, - harness.room, - "host", - "quality-host-client", - ); - const viewer = await openClient(harness.webSocketUrl); - const viewerAuth = await authenticate( - viewer, - harness.room, - "viewer", - "quality-viewer-client", - ); - await host.inbox.next("peer-joined"); - - host.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: viewerAuth.peerId, - payload: { - kind: "description", - connectionId: "quality_connection_first", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await viewer.inbox.next("signal"); - - viewer.socket.send( - JSON.stringify(viewerQualityEvidence("quality_connection_first", 0, 0)), - ); - expect(await host.inbox.next("viewer-quality-evidence")).toEqual({ - ...viewerQualityEvidence("quality_connection_first", 0, 0), - viewerPeerId: viewerAuth.peerId, - parentPeerId: hostAuth.peerId, - }); - - viewer.socket.send( - JSON.stringify(viewerQualityEvidence("quality_connection_first", 0, 1)), - ); - viewer.socket.send( - JSON.stringify( - viewerQualityEvidence("quality_connection_first", 0, 1, 7_000), - ), - ); - await host.inbox.expectNone(30); - - now += 2_000; - viewer.socket.send( - JSON.stringify( - viewerQualityEvidence("quality_connection_first", 0, 1, 7_000), - ), - ); - expect( - (await host.inbox.next("viewer-quality-evidence")).metrics.bitrateKbps, - ).toBe(7_000); - - now += 2_000; - viewer.socket.send( - JSON.stringify( - viewerQualityEvidence("quality_connection_first", 0, 2, 7_000), - ), - ); - expect( - (await host.inbox.next("viewer-quality-evidence")).metrics.bitrateKbps, - ).toBe(7_000); - - viewer.socket.send( - JSON.stringify( - viewerQualityEvidence("quality_connection_stale", 0, 3, 6_500), - ), - ); - viewer.socket.send( - JSON.stringify( - viewerQualityEvidence("quality_connection_first", 1, 3, 6_500), - ), - ); - await host.inbox.expectNone(30); - - host.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: viewerAuth.peerId, - payload: { - kind: "description", - connectionId: "quality_connection_second", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await viewer.inbox.next("signal"); - viewer.socket.send( - JSON.stringify( - viewerQualityEvidence("quality_connection_first", 0, 3, 6_000), - ), - ); - await host.inbox.expectNone(30); - viewer.socket.send( - JSON.stringify( - viewerQualityEvidence("quality_connection_second", 0, 0, 6_000), - ), - ); - expect( - (await host.inbox.next("viewer-quality-evidence")).guard.connectionId, - ).toBe("quality_connection_second"); - - host.socket.send( - JSON.stringify( - viewerQualityEvidence("quality_connection_second", 0, 1, 5_500), - ), - ); - await host.inbox.expectNone(30); - - const replacementViewer = await openClient(harness.webSocketUrl); - const replacementAuth = await authenticate( - replacementViewer, - harness.room, - "viewer", - "quality-viewer-client", - ); - expect(replacementAuth).toMatchObject({ - peerId: viewerAuth.peerId, - connectionId: "quality_connection_second", - }); - expect(await host.inbox.next("peer-joined")).toMatchObject({ - peerId: viewerAuth.peerId, - }); - replacementViewer.socket.send( - JSON.stringify( - viewerQualityEvidence("quality_connection_second", 0, 0, 5_000), - ), - ); - expect(await host.inbox.next("viewer-quality-evidence")).toMatchObject({ - viewerPeerId: viewerAuth.peerId, - guard: { connectionId: "quality_connection_second" }, - sequence: 0, - metrics: { bitrateKbps: 5_000 }, - }); - }); - - it("forwards peer-assisted C to the active parent and opted-in Host", async () => { - let now = 50_000; - const harness = await startHarness({ - peerAssistedMedia: true, - now: () => now, - }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted( - await authenticate( - host, - harness.room, - "host", - "peer-quality-host", - 1, - undefined, - { viewerPresence: true }, - ), - ); - host.inbox.ignore("route-update"); - host.inbox.ignore("media-assignment"); - host.inbox.ignore("viewer-presence"); - const viewer = await openClient(harness.webSocketUrl); - const viewerAuth = peerAssisted( - await authenticate( - viewer, - harness.room, - "viewer", - "peer-quality-viewer", - ), - ); - expect(viewerAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: hostAuth.peerId, - }); - - host.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: viewerAuth.peerId, - payload: { - kind: "description", - connectionId: "peer_quality_connection", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await viewer.inbox.next("signal"); - viewer.socket.send( - JSON.stringify( - viewerQualityEvidence( - "peer_quality_connection", - viewerAuth.routeRevision, - 0, - ), - ), - ); - expect(await host.inbox.next("viewer-quality-evidence")).toMatchObject({ - viewerPeerId: viewerAuth.peerId, - parentPeerId: hostAuth.peerId, - guard: { - connectionId: "peer_quality_connection", - routeRevision: viewerAuth.routeRevision, - }, - }); - - viewer.socket.send( - JSON.stringify( - viewerQualityEvidence( - "peer_quality_connection", - viewerAuth.routeRevision + 1, - 1, - 7_000, - ), - ), - ); - await host.inbox.expectNone(30); - - viewer.inbox.ignore("route-update"); - viewer.inbox.ignore("media-assignment"); - const secondViewer = await openClient(harness.webSocketUrl); - await authenticate( - secondViewer, - harness.room, - "viewer", - "peer-quality-viewer-two", - ); - const relayChild = await openClient(harness.webSocketUrl); - const relayChildAuth = peerAssisted( - await authenticate( - relayChild, - harness.room, - "viewer", - "peer-quality-relay-child", - ), - ); - expect(relayChildAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: viewerAuth.peerId, - }); - - viewer.socket.send( - JSON.stringify( - viewerQualityEvidence( - "peer_quality_connection", - relayChildAuth.routeRevision, - 1, - 6_500, - ), - ), - ); - expect(await host.inbox.next("viewer-quality-evidence")).toMatchObject({ - viewerPeerId: viewerAuth.peerId, - guard: { - connectionId: "peer_quality_connection", - routeRevision: relayChildAuth.routeRevision, - }, - sequence: 1, - }); - - viewer.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: relayChildAuth.peerId, - payload: { - kind: "description", - connectionId: "relay_child_quality_connection", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await relayChild.inbox.next("signal"); - relayChild.socket.send( - JSON.stringify( - viewerQualityEvidence( - "relay_child_quality_connection", - relayChildAuth.routeRevision, - 0, - ), - ), - ); - const relayedEvidence = await viewer.inbox.next("viewer-quality-evidence"); - expect(relayedEvidence).toMatchObject({ - viewerPeerId: relayChildAuth.peerId, - parentPeerId: viewerAuth.peerId, - guard: { - connectionId: "relay_child_quality_connection", - routeRevision: relayChildAuth.routeRevision, - }, - }); - expect(await host.inbox.next("viewer-quality-evidence")).toEqual( - relayedEvidence, - ); - - const replaced = harness.roomStore.connectParticipant({ - roomId: harness.room.roomId, - role: "viewer", - ...(harness.room.viewerGrant - ? { viewerGrant: harness.room.viewerGrant } - : {}), - clientId: "peer-quality-relay-child", - sessionId: "peer-quality-relay-child-replacement-session", - }); - expect(replaced.peerId).toBe(relayChildAuth.peerId); - now += 2_000; - relayChild.socket.send( - JSON.stringify( - viewerQualityEvidence( - "relay_child_quality_connection", - relayChildAuth.routeRevision, - 1, - ), - ), - ); - await viewer.inbox.expectNone(30); - await host.inbox.expectNone(30); - }); - - it("correlates B only from the current parent session and C generation", async () => { - const acceptedParentEvidence = vi.spyOn( - HybridMediaRouter.prototype, - "handleParentEdgeQualityEvidence", - ); - let now = 80_000; - const harness = await startHarness({ - peerAssistedMedia: true, - now: () => now, - }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted( - await authenticate(host, harness.room, "host", "quality-guard-host"), - ); - const viewer = await openClient(harness.webSocketUrl); - const viewerAuth = peerAssisted( - await authenticate(viewer, harness.room, "viewer", "quality-guard-viewer"), - ); - const unrelatedParent = await openClient(harness.webSocketUrl); - const unrelatedParentAuth = peerAssisted( - await authenticate( - unrelatedParent, - harness.room, - "viewer", - "quality-guard-unrelated", - ), - ); - const active = await nextActiveRouteAfter( - viewer, - viewerAuth.routeRevision, - ); - expect(active.assignment.upstream).toEqual({ - kind: "peer", - peerId: hostAuth.peerId, - }); - host.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: viewerAuth.peerId, - payload: { - kind: "description", - connectionId: "quality_guard_connection", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await viewer.inbox.next("signal"); - - let sequence = 0; - const sendBadC = async (parent: TestClient = host) => { - viewer.socket.send( - JSON.stringify( - viewerQualityEvidenceWithMetrics( - "quality_guard_connection", - active.revision, - sequence, - { packetsReceivedDelta: 70, packetsLostDelta: 30 }, - ), - ), - ); - sequence += 1; - return parent.inbox.next("viewer-quality-evidence"); - }; - const parentProof = ( - evidence: Extract, - ) => ({ - type: "parent-edge-quality-evidence" as const, - viewerPeerId: evidence.viewerPeerId, - guard: evidence.guard, - viewerSequence: evidence.sequence, - proof: { - kind: "remote-loss" as const, - packetsSentDelta: 100, - remotePacketsLostDelta: 30, - }, - }); - - let evidence = await sendBadC(); - unrelatedParent.socket.send(JSON.stringify(parentProof(evidence))); - now += 2_000; - - for (const mutate of [ - (proof: ReturnType) => ({ - ...proof, - guard: { ...proof.guard, connectionId: "quality_guard_wrong" }, - }), - (proof: ReturnType) => ({ - ...proof, - guard: { ...proof.guard, routeRevision: proof.guard.routeRevision + 1 }, - }), - (proof: ReturnType) => ({ - ...proof, - viewerSequence: proof.viewerSequence + 1, - }), - ]) { - evidence = await sendBadC(); - host.socket.send(JSON.stringify(mutate(parentProof(evidence)))); - now += 2_000; - } - - evidence = await sendBadC(); - now += 5_001; - host.socket.send(JSON.stringify(parentProof(evidence))); - - now += 2_000; - evidence = await sendBadC(); - const replacementHost = await openClient(harness.webSocketUrl); - const replacementHostAuth = peerAssisted( - await authenticate( - replacementHost, - harness.room, - "host", - "quality-guard-host", - ), - ); - expect(replacementHostAuth.peerId).toBe(hostAuth.peerId); - replacementHost.socket.send(JSON.stringify(parentProof(evidence))); - - expect(acceptedParentEvidence).not.toHaveBeenCalled(); - await expect(viewer.inbox.next("route-update", 40)).rejects.toThrow( - "Timed out", - ); - - now += 2_000; - for (let index = 0; index < 3; index += 1) { - evidence = await sendBadC(replacementHost); - const proof = parentProof(evidence); - replacementHost.socket.send(JSON.stringify(proof)); - if (index === 0) { - replacementHost.socket.send(JSON.stringify(proof)); - } - now += 2_000; - } - const moved = await completePreparedPeerMigration( - viewer, - unrelatedParent, - viewerAuth.peerId, - unrelatedParentAuth.peerId, - active.revision, - "quality_guard_probe", - ); - expect(acceptedParentEvidence).toHaveBeenCalledTimes(3); - expect(moved.assignment.upstream).toEqual({ - kind: "peer", - peerId: unrelatedParentAuth.peerId, - }); - acceptedParentEvidence.mockRestore(); - }); - - it("does not treat a 30 FPS relay child as bad against a 60 FPS setting", async () => { - let now = 90_000; - const fixture = await startRelayRelativeFpsFixture( - "relative_same_fps", - () => now, - ); - await correlateRelayParentInboundFps(fixture, 30); - for (let sequence = 0; sequence < 3; sequence += 1) { - await correlateRelayChildFps(fixture, sequence, 30); - now += 2_000; - } - await expect( - fixture.child.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"); - }); - - it("moves a relay child after three corroborated relative-FPS windows", async () => { - let now = 110_000; - const fixture = await startRelayRelativeFpsFixture( - "relative_peer_move", - () => now, - ); - await correlateRelayParentInboundFps(fixture, 30); - for (let sequence = 0; sequence < 3; sequence += 1) { - await correlateRelayChildFps(fixture, sequence, 10); - now += 2_000; - } - const moved = await completePreparedPeerMigration( - fixture.child, - fixture.alternate, - fixture.childAuth.peerId, - fixture.alternateAuth.peerId, - fixture.routeRevision, - "relative_peer_move_probe", - ); - expect(moved.assignment.upstream).toEqual({ - kind: "peer", - peerId: fixture.alternateAuth.peerId, - }); - - fixture.child.socket.send( - JSON.stringify({ - type: "route-failed", - revision: moved.revision, - phase: "active", - connectionId: "relative_peer_move_probe", - }), - ); - const recovered = await nextActiveRouteAfter(fixture.child, moved.revision); - expect(recovered.assignment.upstream).toEqual({ - kind: "peer", - peerId: fixture.parentAuth.peerId, - }); - }); - - it("uses an eligible Host as the provisional peer parent", async () => { - let now = 120_000; - const fixture = await startRelayRelativeFpsFixture( - "quality_skip_host", - () => now, - { alternateRelayCapacity: 0 }, - ); - fixture.alternate.socket.close(); - const [current] = await Promise.all([ - nextActiveRouteAfter(fixture.child, fixture.routeRevision), - nextActiveRouteAfter(fixture.parent, fixture.routeRevision), - nextActiveRouteAfter(fixture.host, fixture.routeRevision), - ]); - fixture.routeRevision = current.revision; - const advanceWindow = () => { now += 2_000; }; - await confirmRelayChildSevere( - fixture, - fixture.child, - fixture.childConnectionId, - advanceWindow, - ); - const moved = await completePreparedPeerMigration( - fixture.child, - fixture.host, - fixture.childAuth.peerId, - fixture.hostAuth.peerId, - fixture.routeRevision, - "quality_host_probe", - ); - expect(moved.assignment.upstream).toEqual({ - kind: "peer", - peerId: fixture.hostAuth.peerId, - }); - }); - - it("keeps the old edge when the Host upload budget is already full", async () => { - let now = 125_000; - const fixture = await startRelayRelativeFpsFixture( - "quality_only_host", - () => now, - { alternateRelayCapacity: 0 }, - ); - await Promise.all([ - nextActiveRouteRevision(fixture.host, fixture.routeRevision), - nextActiveRouteRevision(fixture.alternate, fixture.routeRevision), - ]); - const advanceWindow = () => { now += 2_000; }; - await confirmRelayChildSevere( - fixture, - fixture.child, - fixture.childConnectionId, - advanceWindow, - ); - await expect(fixture.child.inbox.next("route-update", 40)).rejects.toThrow( - "Timed out", - ); - }); - - it("rolls a Host-parent probe back only for its exact revision and connection", async () => { - let now = 126_000; - const fixture = await startRelayRelativeFpsFixture( - "quality_host_exact_failure", - () => now, - { alternateRelayCapacity: 0 }, - ); - const probeConnectionId = "quality_host_exact_failure_probe"; - const { viewerPrepare } = await prepareHostParentQualityProbe( - fixture, - () => { now += 2_000; }, - probeConnectionId, - ); - - for (const [revision, connectionId] of [ - [viewerPrepare.revision - 1, probeConnectionId], - [viewerPrepare.revision, `${probeConnectionId}-wrong`], - ] as const) { - fixture.child.socket.send(JSON.stringify({ - type: "route-failed", - revision, - phase: "prepare", - connectionId, - })); - } - await Promise.all([ - expect( - fixture.child.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"), - expect( - fixture.host.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"), - ]); - - fixture.child.socket.send(JSON.stringify({ - type: "route-failed", - revision: viewerPrepare.revision, - phase: "prepare", - connectionId: probeConnectionId, - })); - const [viewerRollback, hostRollback] = await Promise.all([ - nextActiveRouteAfter(fixture.child, viewerPrepare.revision), - nextActiveRouteAfter(fixture.host, viewerPrepare.revision), - ]); - expect(viewerRollback.assignment.upstream).toEqual({ - kind: "peer", - peerId: fixture.parentAuth.peerId, - }); - expect(hostRollback.revision).toBe(viewerRollback.revision); - await expect( - sendTestOffer( - fixture.parent, - fixture.child, - fixture.childAuth.peerId, - "quality_host_exact_failure_old_edge", - ), - ).resolves.toMatchObject({ fromPeerId: fixture.parentAuth.peerId }); - }); - - it("invalidates a Host-parent probe when the Host session is replaced", async () => { - let now = 127_000; - const shareGeneration = "quality_host_session_share"; - const fixture = await startRelayRelativeFpsFixture( - "quality_host_session", - () => now, - { alternateRelayCapacity: 0, hostShareGeneration: shareGeneration }, - ); - const { viewerPrepare } = await prepareHostParentQualityProbe( - fixture, - () => { now += 2_000; }, - "quality_host_session_probe", - ); - const oldClosed = new Promise((resolve) => - fixture.host.socket.once("close", (code) => resolve(code)), - ); - const replacement = await openClient(fixture.harness.webSocketUrl); - const replacementAuth = peerAssisted(await authenticate( - replacement, - fixture.harness.room, - "host", - "quality_host_session-host", - 1, - shareGeneration, - )); - expect(await oldClosed).toBe(4001); - expect(replacementAuth.peerId).toBe(fixture.hostAuth.peerId); - expect(replacementAuth.routeRevision).toBeGreaterThan(viewerPrepare.revision); - expect(replacementAuth.routeAssignment.childPeerIds).not.toContain( - fixture.childAuth.peerId, - ); - const rollback = await nextActiveRouteAfter( - fixture.child, - viewerPrepare.revision, - ); - expect(rollback.assignment.upstream).toEqual({ - kind: "peer", - peerId: fixture.parentAuth.peerId, - }); - - fixture.child.socket.send(JSON.stringify({ - type: "route-ready", - revision: viewerPrepare.revision, - phase: "prepare", - })); - await expect( - fixture.child.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"); - }); - - it("invalidates a Host-parent probe when a new share generation starts", async () => { - let now = 127_500; - const fixture = await startRelayRelativeFpsFixture( - "quality_host_share", - () => now, - { - alternateRelayCapacity: 0, - hostShareGeneration: "quality_host_share_a", - }, - ); - const { viewerPrepare } = await prepareHostParentQualityProbe( - fixture, - () => { now += 2_000; }, - "quality_host_share_probe", - ); - const oldClosed = new Promise((resolve) => - fixture.host.socket.once("close", (code) => resolve(code)), - ); - const replacement = await openClient(fixture.harness.webSocketUrl); - const replacementAuth = peerAssisted(await authenticate( - replacement, - fixture.harness.room, - "host", - "quality_host_share-host", - 1, - "quality_host_share_b", - )); - expect(await oldClosed).toBe(4001); - expect(replacementAuth.peerId).toBe(fixture.hostAuth.peerId); - expect(replacementAuth.routeAssignment.childPeerIds).not.toContain( - fixture.childAuth.peerId, - ); - await fixture.child.inbox.next("sharing-stopped"); - const nextShareRoute = await nextActiveRouteRevision( - fixture.child, - replacementAuth.routeRevision, - ); - expect(nextShareRoute.assignment.upstream).toEqual({ - kind: "peer", - peerId: fixture.parentAuth.peerId, - }); - - fixture.child.socket.send(JSON.stringify({ - type: "route-ready", - revision: viewerPrepare.revision, - phase: "prepare", - })); - fixture.child.socket.send(JSON.stringify({ - type: "route-failed", - revision: viewerPrepare.revision, - phase: "prepare", - connectionId: "quality_host_share_probe", - })); - await expect( - fixture.child.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"); - }); - - it("counts an answered Host selected overlay against a quality-probe slot", async () => { - let now = 128_000; - const harness = await startSfuHarness({ - now: () => now, - endpointMediaCopyCapacity: 3, - tokenIssuer: { - async issueToken({ peerId }) { return `token-${peerId}`; }, - }, - selectedEdgeTurn: true, - }); - const active = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "quality-host-selected-budget", - ); - active.viewer.socket.send(JSON.stringify({ - type: "relay-capacity", - downstreamEdges: 0, - })); - const parent = await openClient(harness.webSocketUrl); - const parentAuth = peerAssisted(await authenticate( - parent, - harness.room, - "viewer", - "quality-host-selected-budget-parent", - 1, - )); - expect(parentAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: active.hostAuth.peerId, - }); - const hostFiller = await openClient(harness.webSocketUrl); - const hostFillerAuth = peerAssisted(await authenticate( - hostFiller, - harness.room, - "viewer", - "quality-host-selected-budget-filler", - 0, - )); - expect(hostFillerAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: active.hostAuth.peerId, - }); - const child = await openClient(harness.webSocketUrl); - const childAuth = peerAssisted(await authenticate( - child, - harness.room, - "viewer", - "quality-host-selected-budget-child", - 0, - )); - expect(childAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: parentAuth.peerId, - }); - - hostFiller.socket.close(); - const current = await nextActiveRouteAfter(child, childAuth.routeRevision); - await Promise.all([ - nextActiveRouteRevision(active.host, current.revision), - nextActiveRouteRevision(active.viewer, current.revision), - nextActiveRouteRevision(parent, current.revision), - ]); - const grant = await startPeerSelectedTurn( - active.viewer, - active.host, - current.revision, - ); - active.host.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: active.viewerAuth.peerId, - payload: { - kind: "description", - connectionId: grant.newConnectionId, - description: { type: "offer", sdp: "v=0\r\n" }, - }, - })); - await active.viewer.inbox.next("signal"); - active.viewer.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: active.hostAuth.peerId, - payload: { - kind: "description", - connectionId: grant.newConnectionId, - description: { type: "answer", sdp: "v=0\r\n" }, - }, - })); - await active.host.inbox.next("signal"); - - await sendTestOffer( - active.host, - parent, - parentAuth.peerId, - "quality_host_selected_budget_parent_connection", - ); - const childConnectionId = "quality_host_selected_budget_child_connection"; - await sendTestOffer(parent, child, childAuth.peerId, childConnectionId); - for (let sequence = 0; sequence < 3; sequence += 1) { - child.socket.send(JSON.stringify(viewerQualityEvidenceWithMetrics( - childConnectionId, - current.revision, - sequence, - { framesDecodedDelta: 0 }, - ))); - await correlateParentEdgeQualityEvidence(parent); - now += 2_000; - } - await expect(child.inbox.next("route-update", 40)).rejects.toThrow( - "Timed out", - ); - - active.host.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: active.viewerAuth.peerId, - payload: { - kind: "candidate", - connectionId: grant.newConnectionId, - candidate: null, - }, - })); - await expect(active.viewer.inbox.next("signal")).resolves.toMatchObject({ - fromPeerId: active.hostAuth.peerId, - payload: { connectionId: grant.newConnectionId }, - }); - }); - - it("keeps an SFU-fed Viewer leaf without outbound relay proof", async () => { - let now = 128_000; - const harness = await startSfuHarness({ - now: () => now, - tokenIssuer: { async issueToken({ peerId }) { return `token-${peerId}`; } }, - }); - const active = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "quality_only_host_sfu_root", - ); - active.viewer.socket.send(JSON.stringify({ - type: "relay-capacity", - downstreamEdges: 0, - })); - await new Promise((resolve) => setTimeout(resolve, 0)); - const parent = await openClient(harness.webSocketUrl); - const parentAuth = peerAssisted(await authenticate( - parent, harness.room, "viewer", "quality-sfu-root-parent", 1, - )); - expect(parentAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: active.hostAuth.peerId, - }); - const child = await openClient(harness.webSocketUrl); - const childAuth = peerAssisted(await authenticate( - child, harness.room, "viewer", "quality-sfu-root-child", 0, - )); - expect(childAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: parentAuth.peerId, - }); - - const parentConnectionId = "quality_sfu_root_parent_connection"; - await sendTestOffer( - active.host, - parent, - parentAuth.peerId, - parentConnectionId, - ); - const childConnectionId = "quality_sfu_root_child_connection"; - await sendTestOffer(parent, child, childAuth.peerId, childConnectionId); - const routeRevision = childAuth.routeRevision; - active.viewer.socket.send(JSON.stringify({ - type: "relay-capacity", - downstreamEdges: 2, - })); - await new Promise((resolve) => setTimeout(resolve, 0)); - - for (let sequence = 0; sequence < 3; sequence += 1) { - child.socket.send(JSON.stringify(viewerQualityEvidenceWithMetrics( - childConnectionId, - routeRevision, - sequence, - { framesDecodedDelta: 0 }, - ))); - await correlateParentEdgeQualityEvidence(parent); - now += 2_000; - } - await expect(child.inbox.next("route-update", 40)).rejects.toThrow( - "Timed out", - ); - await expect(active.viewer.inbox.next("signal", 40)).rejects.toThrow( - "Timed out", - ); - }); - - it("keeps the old relay edge when relative FPS has no alternate peer", async () => { - let now = 130_000; - let issuedSfuTokens = 0; - const fixture = await startRelayRelativeFpsFixture( - "relative_no_alternate", - () => now, - { - alternateRelayCapacity: 0, - onSfuTokenIssue: () => { - issuedSfuTokens += 1; - }, - sfu: true, - }, - ); - await correlateRelayParentInboundFps(fixture, 30); - for (let sequence = 0; sequence < 3; sequence += 1) { - await correlateRelayChildFps(fixture, sequence, 10); - now += 2_000; - } - await expect( - fixture.child.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"); - await expect( - fixture.child.inbox.next("sfu-config", 40), - ).rejects.toThrow("Timed out"); - await expect(fixture.child.inbox.next("error", 40)).rejects.toThrow( - "Timed out", - ); - expect(issuedSfuTokens).toBe(0); - - const activeConnectionId = "relative_no_alternate_still_active"; - fixture.parent.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: fixture.childAuth.peerId, - payload: { - kind: "description", - connectionId: activeConnectionId, - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - expect(await fixture.child.inbox.next("signal")).toMatchObject({ - fromPeerId: fixture.parentAuth.peerId, - }); - - for (let sequence = 0; sequence < 3; sequence += 1) { - fixture.child.socket.send( - JSON.stringify( - viewerQualityEvidenceWithMetrics( - activeConnectionId, - fixture.routeRevision, - sequence, - { framesDecodedDelta: 0 }, - ), - ), - ); - await correlateParentEdgeQualityEvidence(fixture.parent); - await new Promise((resolve) => setTimeout(resolve, 0)); - now += 2_000; - } - await expect( - fixture.child.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"); - await expect( - fixture.child.inbox.next("sfu-config", 40), - ).rejects.toThrow("Timed out"); - expect(issuedSfuTokens).toBe(0); - }); - - describe( - "capacity-two relay quality coordination", - () => { - it("keeps a severe child on its old edge when no peer alternate exists", async () => { - let now = 142_000; - const fixture = await startRelayRelativeFpsFixture( - "relay_parent_second_severe", - () => now, - { alternateRelayCapacity: 0, secondChild: true, sfu: true }, - ); - const advanceWindow = () => { - now += 2_000; - }; - await confirmRelayParentChildRelativeFps(fixture, 0, advanceWindow); - await confirmRelayChildSevere( - fixture, - fixture.secondChild!, - fixture.secondChildConnectionId, - advanceWindow, - ); - - await expect( - fixture.secondChild!.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"); - await expect( - fixture.secondChild!.inbox.next("sfu-config", 40), - ).rejects.toThrow("Timed out"); - }, - ); - - it("gives a severe trigger the sole alternate before evacuating siblings", async () => { - let now = 144_000; - let issuedSfuTokens = 0; - const fixture = await startRelayRelativeFpsFixture( - "relay_parent_severe_peer_first", - () => now, - { - alternateRelayCapacity: 0, - onSfuTokenIssue: () => { - issuedSfuTokens += 1; - }, - secondChild: true, - sfu: true, - }, - ); - const advanceWindow = () => { - now += 2_000; - }; - await confirmRelayParentChildRelativeFps(fixture, 0, advanceWindow); - fixture.alternate.socket.send(JSON.stringify({ - type: "relay-capacity", - downstreamEdges: 1, - })); - await new Promise((resolve) => setTimeout(resolve, 0)); - await confirmRelayChildSevere( - fixture, - fixture.secondChild!, - fixture.secondChildConnectionId, - advanceWindow, - ); - - const triggerMoved = await completePreparedPeerMigration( - fixture.secondChild!, - fixture.alternate, - fixture.secondChildAuth!.peerId, - fixture.alternateAuth.peerId, - fixture.routeRevision, - "relay_parent_severe_peer_probe", - ); - expect(triggerMoved.assignment.upstream).toEqual({ - kind: "peer", - peerId: fixture.alternateAuth.peerId, - }); - const siblingStayed = await nextActiveRouteRevision( - fixture.child, - triggerMoved.revision, - ); - expect(siblingStayed.assignment.upstream).toEqual({ - kind: "peer", - peerId: fixture.parentAuth.peerId, - }); - expect(issuedSfuTokens).toBe(0); - }); - - it("does not escalate soft quality recovery to SFU or selected TURN", async () => { - let now = 144_500; - let issuedSfuTokens = 0; - const fixture = await startRelayRelativeFpsFixture( - "relay_parent_severe_selected_turn", - () => now, - { - alternateRelayCapacity: 0, - onSfuTokenIssue: () => { issuedSfuTokens += 1; }, - secondChild: true, - selectedEdgeTurn: true, - sfu: true, - }, - ); - const advanceWindow = () => { - now += 2_000; - }; - await confirmRelayParentChildRelativeFps(fixture, 0, advanceWindow); - await confirmRelayChildSevere( - fixture, - fixture.secondChild!, - fixture.secondChildConnectionId, - advanceWindow, - ); - await expect( - fixture.secondChild!.inbox.next("sfu-config", 40), - ).rejects.toThrow("Timed out"); - await expect( - fixture.secondChild!.inbox.next("selected-edge-turn", 40), - ).rejects.toThrow("Timed out"); - expect(issuedSfuTokens).toBe(0); - }); - - it("does not drain a relay after two child-scoped quality events", async () => { - let now = 145_000; - let issuedSfuTokens = 0; - const fixture = await startRelayRelativeFpsFixture( - "relay_parent_child_scoped_quality", - () => now, - { - alternateRelayCapacity: 0, - onSfuTokenIssue: () => { - issuedSfuTokens += 1; - }, - secondChild: true, - sfu: true, - }, - ); - const secondChild = fixture.secondChild!; - - const advanceWindow = () => { - now += 2_000; - }; - await confirmRelayParentChildRelativeFps(fixture, 0, advanceWindow); - await confirmRelayParentChildRelativeFps(fixture, 1, advanceWindow); - - expect(issuedSfuTokens).toBe(0); - expect(await sendTestOffer( - fixture.parent, - secondChild, - fixture.secondChildAuth!.peerId, - "relay_parent_child_scoped_still_active", - )).toMatchObject({ - fromPeerId: fixture.parentAuth.peerId, - }); - - fixture.child.socket.close(); - await new Promise((resolve) => setTimeout(resolve, 70)); - fixture.parent.socket.send( - JSON.stringify({ type: "relay-capacity", downstreamEdges: 2 }), - ); - await new Promise((resolve) => setTimeout(resolve, 0)); - const pending = await openClient(fixture.harness.webSocketUrl); - const pendingAuth = peerAssisted( - await authenticate( - pending, - fixture.harness.room, - "viewer", - "relay-parent-child-scoped-pending", - 0, - ), - ); - expect(pendingAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: fixture.parentAuth.peerId, - }); - }); - - it("serializes child-scoped migrations through the room cooldown", async () => { - let now = 180_000; - let issuedSfuTokens = 0; - const fixture = await startRelayRelativeFpsFixture( - "relay_parent_peer_only", - () => now, - { - alternateRelayCapacity: 1, - onSfuTokenIssue: () => { - issuedSfuTokens += 1; - }, - secondChild: true, - sfu: true, - }, - ); - const secondChild = fixture.secondChild!; - - const advanceWindow = () => { - now += 2_000; - }; - await confirmRelayParentChildRelativeFps(fixture, 0, advanceWindow); - const firstMoved = await completePreparedPeerMigration( - fixture.child, - fixture.alternate, - fixture.childAuth.peerId, - fixture.alternateAuth.peerId, - fixture.routeRevision, - "relay_parent_peer_only_probe", - ); - expect(firstMoved.assignment.upstream).toEqual({ - kind: "peer", - peerId: fixture.alternateAuth.peerId, - }); - fixture.routeRevision = firstMoved.revision; - const secondAtFirstMove = await nextActiveRouteRevision( - secondChild, - firstMoved.revision, - ); - expect(secondAtFirstMove.assignment.upstream).toEqual({ - kind: "peer", - peerId: fixture.parentAuth.peerId, - }); - await confirmRelayParentChildRelativeFps(fixture, 1, advanceWindow); - await expect(secondChild.inbox.next("route-update", 40)).rejects.toThrow( - "Timed out", - ); - - for (let sequence = 2; sequence < 5; sequence += 1) { - fixture.parent.socket.send(JSON.stringify(viewerQualityEvidenceWithMetrics( - fixture.parentConnectionId, - fixture.routeRevision, - sequence, - { framesDecodedDelta: 0 }, - ))); - await correlateParentEdgeQualityEvidence(fixture.host); - now += 2_000; - } - expect(issuedSfuTokens).toBe(0); - await expect( - fixture.parent.inbox.next("sfu-config", 40), - ).rejects.toThrow("Timed out"); - }); - - it("allows only one peer quality migration per room", async () => { - let now = 200_000; - const fixture = await startRelayRelativeFpsFixture( - "relay_parent_pending_severe", - () => now, - { - alternateRelayCapacity: 1, - secondChild: true, - sfu: true, - }, - ); - const advanceWindow = () => { - now += 2_000; - }; - await confirmRelayChildSevere( - fixture, - fixture.child, - fixture.childConnectionId, - advanceWindow, - ); - await confirmRelayChildSevere( - fixture, - fixture.secondChild!, - fixture.secondChildConnectionId, - advanceWindow, - ); - await expect( - fixture.secondChild!.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"); - const childActive = await completePreparedPeerMigration( - fixture.child, - fixture.alternate, - fixture.childAuth.peerId, - fixture.alternateAuth.peerId, - fixture.routeRevision, - "relay_parent_pending_probe", - ); - expect(childActive.assignment.upstream).toEqual({ - kind: "peer", - peerId: fixture.alternateAuth.peerId, - }); - const siblingStayed = await nextActiveRouteRevision( - fixture.secondChild!, - childActive.revision, - ); - expect(siblingStayed.assignment.upstream).toEqual({ - kind: "peer", - peerId: fixture.parentAuth.peerId, - }); - }); - - it("does not combine confirmed child events across the five-second gap", async () => { - let now = 220_000; - const fixture = await startRelayRelativeFpsFixture( - "relay_parent_stale_gap", - () => now, - { alternateRelayCapacity: 0, secondChild: true }, - ); - const secondChild = fixture.secondChild!; - - const advanceWindow = () => { - now += 2_000; - }; - await confirmRelayParentChildRelativeFps(fixture, 0, advanceWindow); - now += 5_001; - fixture.alternate.socket.send( - JSON.stringify({ type: "relay-capacity", downstreamEdges: 2 }), - ); - await new Promise((resolve) => setTimeout(resolve, 0)); - await confirmRelayParentChildRelativeFps(fixture, 1, advanceWindow); - - const secondMoved = await completePreparedPeerMigration( - secondChild, - fixture.alternate, - fixture.secondChildAuth!.peerId, - fixture.alternateAuth.peerId, - fixture.routeRevision, - "relay_parent_stale_gap_probe", - ); - expect(secondMoved.assignment.upstream).toEqual({ - kind: "peer", - peerId: fixture.alternateAuth.peerId, - }); - const firstStayed = await nextActiveRouteRevision( - fixture.child, - secondMoved.revision, - ); - expect(firstStayed.assignment.upstream).toEqual({ - kind: "peer", - peerId: fixture.parentAuth.peerId, - }); - }); - - it.each(["session", "share"] as const)( - "does not combine confirmed children across a new parent %s", - async (boundary) => { - let now = boundary === "session" ? 250_000 : 280_000; - const prefix = `relay_parent_stale_${boundary}`; - const fixture = await startRelayRelativeFpsFixture(prefix, () => now, { - alternateRelayCapacity: 0, - secondChild: true, - }); - const secondChild = fixture.secondChild!; - - const advanceWindow = () => { - now += 2_000; - }; - await confirmRelayParentChildRelativeFps(fixture, 0, advanceWindow); - - if (boundary === "session") { - const replacement = await openClient(fixture.harness.webSocketUrl); - const replacementAuth = peerAssisted( - await authenticate( - replacement, - fixture.harness.room, - "viewer", - `${prefix}-parent`, - 2, - ), - ); - expect(replacementAuth.peerId).toBe(fixture.parentAuth.peerId); - fixture.parent = replacement; - fixture.parentAuth = replacementAuth; - fixture.routeRevision = replacementAuth.routeRevision; - } else { - const replacement = await openClient(fixture.harness.webSocketUrl); - const replacementAuth = peerAssisted( - await authenticate( - replacement, - fixture.harness.room, - "host", - `${prefix}-host`, - 1, - `${prefix}-next-share`, - ), - ); - expect(replacementAuth.peerId).toBe(fixture.hostAuth.peerId); - fixture.host = replacement; - fixture.hostAuth = replacementAuth; - fixture.routeRevision = replacementAuth.routeRevision; - } - await reconnectRelayQualityEdges(fixture, `${prefix}_replacement`); - fixture.alternate.socket.send( - JSON.stringify({ type: "relay-capacity", downstreamEdges: 2 }), - ); - await new Promise((resolve) => setTimeout(resolve, 0)); - await confirmRelayParentChildRelativeFps(fixture, 1, advanceWindow); - - const secondMoved = await completePreparedPeerMigration( - secondChild, - fixture.alternate, - fixture.secondChildAuth!.peerId, - fixture.alternateAuth.peerId, - fixture.routeRevision, - `${prefix}_probe`, - ); - expect(secondMoved.assignment.upstream).toEqual({ - kind: "peer", - peerId: fixture.alternateAuth.peerId, - }); - const firstChildSignal = await sendTestOffer( - fixture.parent, - fixture.child, - fixture.childAuth.peerId, - `${prefix}_first_child_still_active`, - ); - expect(firstChildSignal).toMatchObject({ - fromPeerId: fixture.parentAuth.peerId, - }); - }, - ); - - }); - - it("does not apply relative-FPS reparenting to a Host-direct edge", async () => { - let now = 150_000; - const harness = await startHarness({ - peerAssistedMedia: true, - now: () => now, - }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted( - await authenticate(host, harness.room, "host", "relative_host_direct_host"), - ); - const viewer = await openClient(harness.webSocketUrl); - const viewerAuth = peerAssisted( - await authenticate( - viewer, - harness.room, - "viewer", - "relative_host_direct_viewer", - 0, - ), - ); - const alternate = await openClient(harness.webSocketUrl); - const alternateAuth = peerAssisted( - await authenticate( - alternate, - harness.room, - "viewer", - "relative_host_direct_alternate", - 1, - ), - ); - await nextActiveRouteRevision(viewer, alternateAuth.routeRevision); - expect(viewerAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: hostAuth.peerId, - }); - host.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: viewerAuth.peerId, - payload: { - kind: "description", - connectionId: "relative_host_direct_connection", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await viewer.inbox.next("signal"); - for (let sequence = 0; sequence < 3; sequence += 1) { - viewer.socket.send( - JSON.stringify( - viewerQualityEvidenceWithMetrics( - "relative_host_direct_connection", - alternateAuth.routeRevision, - sequence, - { framesPerSecond: 10 }, - ), - ), - ); - await correlateParentEdgeQualityEvidence(host); - now += 2_000; - } - await expect(viewer.inbox.next("route-update", 40)).rejects.toThrow( - "Timed out", - ); - }); - - it("requires fresh correlated inbound FPS from the relay parent", async () => { - let now = 170_000; - const fixture = await startRelayRelativeFpsFixture( - "relative_parent_freshness", - () => now, - ); - for (let sequence = 0; sequence < 3; sequence += 1) { - await correlateRelayChildFps(fixture, sequence, 10); - now += 2_000; - } - await expect( - fixture.child.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"); - - await correlateRelayParentInboundFps(fixture, 30); - now += 5_001; - for (let sequence = 3; sequence < 6; sequence += 1) { - await correlateRelayChildFps(fixture, sequence, 10); - now += 2_000; - } - await expect( - fixture.child.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"); - }); - - it.each(["connection", "revision", "session"] as const)( - "rejects relative FPS after the relay parent's %s identity changes", - async (changedIdentity) => { - let now = 180_000; - const prefix = `relative_parent_${changedIdentity}`; - const fixture = await startRelayRelativeFpsFixture(prefix, () => now); - await correlateRelayParentInboundFps(fixture, 30); - let routeRevision = fixture.routeRevision; - - if (changedIdentity === "connection") { - fixture.host.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: fixture.parentAuth.peerId, - payload: { - kind: "description", - connectionId: `${prefix}_replacement_connection`, - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await fixture.parent.inbox.next("signal"); - } else if (changedIdentity === "revision") { - const extra = await openClient(fixture.harness.webSocketUrl); - const extraAuth = peerAssisted( - await authenticate( - extra, - fixture.harness.room, - "viewer", - `${prefix}-extra`, - 0, - ), - ); - routeRevision = extraAuth.routeRevision; - await nextActiveRouteRevision(fixture.child, routeRevision); - } else { - const replacement = await openClient(fixture.harness.webSocketUrl); - const replacementAuth = peerAssisted( - await authenticate( - replacement, - fixture.harness.room, - "viewer", - `${prefix}-parent`, - 1, - ), - ); - expect(replacementAuth.peerId).toBe(fixture.parentAuth.peerId); - routeRevision = replacementAuth.routeRevision; - fixture.parent = replacement; - } - - for (let sequence = 0; sequence < 3; sequence += 1) { - fixture.child.socket.send( - JSON.stringify( - viewerQualityEvidenceWithMetrics( - fixture.childConnectionId, - routeRevision, - sequence, - { framesPerSecond: 10 }, - ), - ), - ); - await correlateParentEdgeQualityEvidence(fixture.parent, { - kind: "sender-limited", - packetsSentDelta: 1_500, - reason: "bandwidth", - }); - await new Promise((resolve) => setTimeout(resolve, 0)); - now += 2_000; - } - await expect( - fixture.child.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"); - }, - ); - - it("keeps severe zero-decode quality on the old edge without an alternate", async () => { - let now = 190_000; - const harness = await startSfuHarness({ - now: () => now, - tokenIssuer: { issueToken: async ({ peerId }) => `token-${peerId}` }, - }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted( - await authenticate(host, harness.room, "host", "severe_sfu_host"), - ); - const viewer = await openClient(harness.webSocketUrl); - const viewerAuth = peerAssisted( - await authenticate(viewer, harness.room, "viewer", "severe_sfu_viewer", 0), - ); - host.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: viewerAuth.peerId, - payload: { - kind: "description", - connectionId: "severe_sfu_connection", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await viewer.inbox.next("signal"); - for (let sequence = 0; sequence < 3; sequence += 1) { - viewer.socket.send( - JSON.stringify( - viewerQualityEvidenceWithMetrics( - "severe_sfu_connection", - viewerAuth.routeRevision, - sequence, - { framesDecodedDelta: 0 }, - ), - ), - ); - await correlateParentEdgeQualityEvidence(host); - now += 2_000; - } - await expect(viewer.inbox.next("route-update", 40)).rejects.toThrow( - "Timed out", - ); - await expect(viewer.inbox.next("sfu-config", 40)).rejects.toThrow( - "Timed out", - ); - expect(hostAuth.routeAssignment.upstream).toEqual({ kind: "none" }); - }); - - it("reparents only a viewer subtree after three hard quality windows", async () => { - let now = 100_000; - const harness = await startHarness({ - peerAssistedMedia: true, - now: () => now, - }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted( - await authenticate(host, harness.room, "host", "quality-route-host"), - ); - host.inbox.ignore("route-update"); - host.inbox.ignore("media-assignment"); - - const parentA = await openClient(harness.webSocketUrl); - const parentAAuth = peerAssisted( - await authenticate( - parentA, - harness.room, - "viewer", - "quality-route-parent-a", - ), - ); - expect(parentAAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: hostAuth.peerId, - }); - const parentB = await openClient(harness.webSocketUrl); - const parentBAuth = peerAssisted( - await authenticate( - parentB, - harness.room, - "viewer", - "quality-route-parent-b", - ), - ); - expect(parentBAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: hostAuth.peerId, - }); - const subtreeRoot = await openClient(harness.webSocketUrl); - const subtreeRootAuth = peerAssisted( - await authenticate( - subtreeRoot, - harness.room, - "viewer", - "quality-route-subtree-root", - ), - ); - expect(subtreeRootAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: parentAAuth.peerId, - }); - const alternateParent = await openClient(harness.webSocketUrl); - const alternateParentAuth = peerAssisted( - await authenticate( - alternateParent, - harness.room, - "viewer", - "quality-route-alternate-parent", - ), - ); - expect(alternateParentAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: parentBAuth.peerId, - }); - const subtreeLeaf = await openClient(harness.webSocketUrl); - const subtreeLeafAuth = peerAssisted( - await authenticate( - subtreeLeaf, - harness.room, - "viewer", - "quality-route-subtree-leaf", - 0, - ), - ); - expect(subtreeLeafAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: subtreeRootAuth.peerId, - }); - await nextActiveRouteRevision( - subtreeRoot, - subtreeLeafAuth.routeRevision, - ); - - parentA.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: subtreeRootAuth.peerId, - payload: { - kind: "description", - connectionId: "quality_route_parent_a", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await subtreeRoot.inbox.next("signal"); - - for (let sequence = 0; sequence < 3; sequence += 1) { - subtreeRoot.socket.send( - JSON.stringify( - viewerQualityEvidenceWithMetrics( - "quality_route_stale", - subtreeLeafAuth.routeRevision, - sequence, - { framesDecodedDelta: 0 }, - ), - ), - ); - } - await expect( - subtreeRoot.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"); - - for (let sequence = 0; sequence < 3; sequence += 1) { - subtreeRoot.socket.send( - JSON.stringify( - viewerQualityEvidenceWithMetrics( - "quality_route_parent_a", - subtreeLeafAuth.routeRevision, - sequence, - { framesDecodedDelta: 0 }, - ), - ), - ); - await parentA.inbox.next("viewer-quality-evidence"); - now += 2_000; - } - await expect( - subtreeRoot.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"); - - for (let sequence = 3; sequence < 6; sequence += 1) { - subtreeRoot.socket.send( - JSON.stringify( - viewerQualityEvidenceWithMetrics( - "quality_route_parent_a", - subtreeLeafAuth.routeRevision, - sequence, - { framesDecodedDelta: 0 }, - ), - ), - ); - await correlateParentEdgeQualityEvidence(parentA, { - kind: "sending", - packetsSentDelta: 1_500, - }); - now += 2_000; - } - await expect( - subtreeRoot.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"); - - subtreeRoot.socket.send( - JSON.stringify( - viewerQualityEvidenceWithMetrics( - "quality_route_parent_a", - subtreeLeafAuth.routeRevision, - 6, - { packetsReceivedDelta: 70, packetsLostDelta: 30 }, - ), - ), - ); - await correlateParentEdgeQualityEvidence(parentA); - now += 2_000; - subtreeRoot.socket.send( - JSON.stringify( - viewerQualityEvidence( - "quality_route_parent_a", - subtreeLeafAuth.routeRevision, - 7, - ), - ), - ); - await correlateParentEdgeQualityEvidence(parentA); - now += 2_000; - subtreeRoot.socket.send( - JSON.stringify( - viewerQualityEvidenceWithMetrics( - "quality_route_parent_a", - subtreeLeafAuth.routeRevision, - 8, - { freezeDurationMsDelta: 1_000 }, - ), - ), - ); - await correlateParentEdgeQualityEvidence(parentA, { - kind: "sender-limited", - packetsSentDelta: 1_500, - reason: "cpu", - }); - now += 2_000; - subtreeRoot.socket.send( - JSON.stringify( - viewerQualityEvidenceWithMetrics( - "quality_route_parent_a", - subtreeLeafAuth.routeRevision, - 9, - { freezeDurationMsDelta: 1_000 }, - ), - ), - ); - await correlateParentEdgeQualityEvidence(parentA, { - kind: "sender-limited", - packetsSentDelta: 1_500, - reason: "bandwidth", - }); - await expect( - subtreeRoot.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"); - - now += 2_000; - subtreeRoot.socket.send( - JSON.stringify( - viewerQualityEvidenceWithMetrics( - "quality_route_parent_a", - subtreeLeafAuth.routeRevision, - 10, - { freezeDurationMsDelta: 1_000 }, - ), - ), - ); - await correlateParentEdgeQualityEvidence(parentA, { - kind: "sender-limited", - packetsSentDelta: 1_500, - reason: "cpu", - }); - const movedRoot = await completePreparedPeerMigration( - subtreeRoot, - alternateParent, - subtreeRootAuth.peerId, - alternateParentAuth.peerId, - subtreeLeafAuth.routeRevision, - "quality_route_probe", - ); - expect(movedRoot.assignment.upstream).toEqual({ - kind: "peer", - peerId: alternateParentAuth.peerId, - }); - const movedLeaf = await nextActiveRouteAfter( - subtreeLeaf, - subtreeLeafAuth.routeRevision, - ); - expect(movedLeaf).toMatchObject({ - revision: movedRoot.revision, - assignment: { - upstream: { kind: "peer", peerId: subtreeRootAuth.peerId }, - }, - }); - const unchangedBranch = await nextActiveRouteAfter( - alternateParent, - subtreeLeafAuth.routeRevision, - ); - expect(unchangedBranch).toMatchObject({ - revision: movedRoot.revision, - assignment: { - upstream: { kind: "peer", peerId: parentBAuth.peerId }, - childPeerIds: [subtreeRootAuth.peerId], - }, - }); - const unchangedBranchRoot = await nextActiveRouteAfter( - parentB, - subtreeLeafAuth.routeRevision, - ); - expect(unchangedBranchRoot).toMatchObject({ - revision: movedRoot.revision, - assignment: { - upstream: { kind: "peer", peerId: hostAuth.peerId }, - childPeerIds: [alternateParentAuth.peerId], - }, - }); - - subtreeRoot.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: subtreeLeafAuth.peerId, - payload: { - kind: "description", - connectionId: "quality_route_leaf", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await subtreeLeaf.inbox.next("signal"); - for (let sequence = 0; sequence < 3; sequence += 1) { - now += 2_000; - subtreeLeaf.socket.send( - JSON.stringify( - viewerQualityEvidenceWithMetrics( - "quality_route_leaf", - movedRoot.revision, - sequence, - { framesDecodedDelta: 0 }, - ), - ), - ); - await correlateParentEdgeQualityEvidence(subtreeRoot); - } - await expect( - subtreeLeaf.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"); - await expect( - subtreeLeaf.inbox.next("error", 40), - ).rejects.toThrow("Timed out"); - - alternateParent.socket.close(); - const reattachedRoot = await nextActiveRouteAfter( - subtreeRoot, - movedRoot.revision, - ); - expect(reattachedRoot.assignment.upstream).toEqual({ - kind: "peer", - peerId: parentAAuth.peerId, - }); - - parentA.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: subtreeRootAuth.peerId, - payload: { - kind: "description", - connectionId: "quality_route_reattached", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await subtreeRoot.inbox.next("signal"); - subtreeRoot.socket.send( - JSON.stringify({ - type: "route-failed", - revision: reattachedRoot.revision, - phase: "active", - connectionId: "quality_route_reattached", - }), - ); - const recoveredRoot = await nextActiveRouteAfter( - subtreeRoot, - reattachedRoot.revision, - ); - expect(recoveredRoot.assignment.upstream).toEqual({ - kind: "peer", - peerId: parentBAuth.peerId, - }); - }); - - it("resets a hard-quality streak after a five-second evidence gap", async () => { - let now = 200_000; - const harness = await startHarness({ - peerAssistedMedia: true, - now: () => now, - }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted( - await authenticate(host, harness.room, "host", "quality-gap-host"), - ); - const viewer = await openClient(harness.webSocketUrl); - const viewerAuth = peerAssisted( - await authenticate(viewer, harness.room, "viewer", "quality-gap-viewer"), - ); - host.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: viewerAuth.peerId, - payload: { - kind: "description", - connectionId: "quality_gap_connection", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await viewer.inbox.next("signal"); - - const sendBadWindow = async (sequence: number) => { - viewer.socket.send( - JSON.stringify( - viewerQualityEvidenceWithMetrics( - "quality_gap_connection", - viewerAuth.routeRevision, - sequence, - { framesDecodedDelta: 0 }, - ), - ), - ); - await correlateParentEdgeQualityEvidence(host); - }; - await sendBadWindow(0); - now += 2_000; - await sendBadWindow(1); - now += 5_001; - await sendBadWindow(2); - await expect(viewer.inbox.next("error", 40)).rejects.toThrow("Timed out"); - now += 2_000; - await sendBadWindow(3); - await expect(viewer.inbox.next("error", 40)).rejects.toThrow("Timed out"); - now += 2_000; - await sendBadWindow(4); - await expect(viewer.inbox.next("error", 40)).rejects.toThrow("Timed out"); - expect(viewerAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: hostAuth.peerId, - }); - }); - - it("breaks a hard-quality streak when newer C replaces an unmatched window", async () => { - let now = 225_000; - const harness = await startHarness({ - peerAssistedMedia: true, - now: () => now, - }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted( - await authenticate(host, harness.room, "host", "quality-pending-host"), - ); - const viewer = await openClient(harness.webSocketUrl); - const viewerAuth = peerAssisted( - await authenticate( - viewer, - harness.room, - "viewer", - "quality-pending-viewer", - ), - ); - host.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: viewerAuth.peerId, - payload: { - kind: "description", - connectionId: "quality_pending_connection", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await viewer.inbox.next("signal"); - - const sendBadC = (sequence: number) => { - viewer.socket.send( - JSON.stringify( - viewerQualityEvidenceWithMetrics( - "quality_pending_connection", - viewerAuth.routeRevision, - sequence, - { framesDecodedDelta: 0 }, - ), - ), - ); - }; - - sendBadC(0); - await correlateParentEdgeQualityEvidence(host); - now += 2_000; - - sendBadC(1); - await host.inbox.next("viewer-quality-evidence"); - now += 2_000; - - sendBadC(2); - await correlateParentEdgeQualityEvidence(host); - now += 2_000; - sendBadC(3); - await correlateParentEdgeQualityEvidence(host); - await expect(viewer.inbox.next("error", 40)).rejects.toThrow("Timed out"); - - now += 2_000; - sendBadC(4); - await correlateParentEdgeQualityEvidence(host); - await expect(viewer.inbox.next("error", 40)).rejects.toThrow("Timed out"); - expect(viewerAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: hostAuth.peerId, - }); - }); - - it("keeps the old edge when a peer quality probe fails", async () => { - let now = 250_000; - const fixture = await startRelayRelativeFpsFixture( - "quality_probe_failure", - () => now, - { alternateRelayCapacity: 1 }, - ); - const advanceWindow = () => { now += 2_000; }; - await confirmRelayChildSevere( - fixture, - fixture.child, - fixture.childConnectionId, - advanceWindow, - ); - const [viewerPrepare, parentPrepare] = await Promise.all([ - nextPreparedRoute(fixture.child), - nextPreparedRoute(fixture.alternate), - ]); - expect(parentPrepare.revision).toBe(viewerPrepare.revision); - await sendTestOffer( - fixture.parent, - fixture.child, - fixture.childAuth.peerId, - fixture.childConnectionId, - ); - const probeConnectionId = "quality_probe_failure_candidate"; - await sendTestOffer( - fixture.alternate, - fixture.child, - fixture.childAuth.peerId, - probeConnectionId, - ); - fixture.child.socket.send(JSON.stringify({ - type: "route-failed", - revision: viewerPrepare.revision, - phase: "prepare", - connectionId: probeConnectionId, - })); - const rollback = await nextActiveRouteAfter( - fixture.child, - viewerPrepare.revision, - ); - expect(rollback.assignment.upstream).toEqual({ - kind: "peer", - peerId: fixture.parentAuth.peerId, - }); - await sendTestOffer( - fixture.parent, - fixture.child, - fixture.childAuth.peerId, - "quality_probe_failure_old_edge", - ); - }); - - it("keeps the old edge when a peer quality probe times out", async () => { - let now = 300_000; - let issuedSfuTokens = 0; - const fixture = await startRelayRelativeFpsFixture( - "quality_probe_timeout", - () => now, - { - alternateRelayCapacity: 1, - onSfuTokenIssue: () => { issuedSfuTokens += 1; }, - prepareTimeoutMs: 20, - sfu: true, - }, - ); - const advanceWindow = () => { now += 2_000; }; - await confirmRelayChildSevere( - fixture, - fixture.child, - fixture.childConnectionId, - advanceWindow, - ); - const [viewerPrepare, parentPrepare] = await Promise.all([ - nextPreparedRoute(fixture.child), - nextPreparedRoute(fixture.alternate), - ]); - expect(parentPrepare.revision).toBe(viewerPrepare.revision); - await sendTestOffer( - fixture.parent, - fixture.child, - fixture.childAuth.peerId, - fixture.childConnectionId, - ); - const rollback = await nextActiveRouteAfter( - fixture.child, - viewerPrepare.revision, - ); - expect(rollback.assignment.upstream).toEqual({ - kind: "peer", - peerId: fixture.parentAuth.peerId, - }); - await expect(fixture.child.inbox.next("sfu-config", 40)).rejects.toThrow( - "Timed out", - ); - expect(issuedSfuTokens).toBe(0); - }); - - it("lets another viewer hard failure preempt a soft peer probe", async () => { - let now = 320_000; - const fixture = await startRelayRelativeFpsFixture( - "quality_probe_hard_preemption", - () => now, - { alternateRelayCapacity: 1 }, - ); - const advanceWindow = () => { now += 2_000; }; - await confirmRelayChildSevere( - fixture, - fixture.child, - fixture.childConnectionId, - advanceWindow, - ); - const [viewerPrepare, parentPrepare] = await Promise.all([ - nextPreparedRoute(fixture.child), - nextPreparedRoute(fixture.alternate), - ]); - expect(parentPrepare.revision).toBe(viewerPrepare.revision); - - fixture.parent.socket.send(JSON.stringify({ - type: "route-failed", - revision: fixture.routeRevision, - phase: "active", - connectionId: fixture.parentConnectionId, - })); - const [softRollback, candidateRollback, hardRollback] = await Promise.all([ - nextActiveRouteAfter(fixture.child, viewerPrepare.revision), - nextActiveRouteAfter(fixture.alternate, viewerPrepare.revision), - nextActiveRouteAfter(fixture.parent, viewerPrepare.revision), - ]); - expect(softRollback.assignment.upstream).toEqual({ - kind: "peer", - peerId: fixture.parentAuth.peerId, - }); - expect(candidateRollback.revision).toBe(softRollback.revision); - expect(hardRollback.revision).toBe(softRollback.revision); - - const hardMoved = await nextActiveRouteAfter( - fixture.parent, - hardRollback.revision, - ); - expect(hardMoved.assignment.upstream).toEqual({ - kind: "peer", - peerId: fixture.alternateAuth.peerId, - }); - }); - - it("rejects viewer C evidence above the byte cap", async () => { - const harness = await startHarness(); - const viewer = await openClient(harness.webSocketUrl); - await authenticate( - viewer, - harness.room, - "viewer", - "oversized-quality-viewer", - ); - const closed = new Promise((resolve) => - viewer.socket.once("close", (code) => resolve(code)), - ); - const valid = JSON.stringify( - viewerQualityEvidence("oversized_quality_connection", 0, 0), - ); - - viewer.socket.send( - `${valid}${" ".repeat(MAX_VIEWER_QUALITY_EVIDENCE_BYTES + 1)}`, - ); - - expect(await viewer.inbox.next("error")).toMatchObject({ - code: "INVALID_MESSAGE", - }); - expect(await closed).toBe(1008); - }); - - it("rejects parent B evidence above the byte cap", async () => { - const harness = await startHarness(); - const host = await openClient(harness.webSocketUrl); - await authenticate(host, harness.room, "host", "oversized-parent-host"); - const closed = new Promise((resolve) => - host.socket.once("close", (code) => resolve(code)), - ); - const valid = JSON.stringify({ - type: "parent-edge-quality-evidence", - viewerPeerId: "oversized_parent_viewer", - guard: { - connectionId: "oversized_parent_connection", - routeRevision: 0, - }, - viewerSequence: 0, - proof: { kind: "sending", packetsSentDelta: 1 }, - }); - - host.socket.send( - `${valid}${" ".repeat(MAX_PARENT_EDGE_QUALITY_EVIDENCE_BYTES + 1)}`, - ); - expect(await host.inbox.next("error")).toMatchObject({ - code: "INVALID_MESSAGE", - }); - expect(await closed).toBe(1008); - }); - - it("routes offer and answer only between the host and the targeted viewer", async () => { - const harness = await startHarness(); - const secondRoom = harness.roomStore.createRoom(); - const host = await openClient(harness.webSocketUrl); - const hostAuth = await authenticate(host, harness.room, "host", "host-client-a"); - const viewer = await openClient(harness.webSocketUrl); - const viewerAuth = await authenticate(viewer, harness.room, "viewer", "viewer-client-a"); - await host.inbox.next("peer-joined"); - const foreignViewer = await openClient(harness.webSocketUrl); - const foreignAuth = await authenticate( - foreignViewer, - secondRoom, - "viewer", - "viewer-client-b", - ); - - host.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: viewerAuth.peerId, - payload: { - kind: "description", - connectionId: "connection-a", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - expect(await viewer.inbox.next("signal")).toMatchObject({ - fromPeerId: hostAuth.peerId, - payload: { description: { type: "offer" } }, - }); - - viewer.socket.send( - JSON.stringify({ - type: "signal", - payload: { - kind: "description", - connectionId: "connection-a", - description: { type: "answer", sdp: "v=0\r\n" }, - }, - }), - ); - expect(await host.inbox.next("signal")).toMatchObject({ - fromPeerId: viewerAuth.peerId, - payload: { description: { type: "answer" } }, - }); - - viewer.socket.send( - JSON.stringify({ - type: "signal", - payload: { - kind: "description", - connectionId: "connection-b", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - expect((await viewer.inbox.next("error")).code).toBe("FORBIDDEN"); - - host.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: foreignAuth.peerId, - payload: { - kind: "candidate", - connectionId: "connection-a", - candidate: null, - }, - }), - ); - expect((await host.inbox.next("error")).code).toBe("PEER_NOT_FOUND"); - await foreignViewer.inbox.expectNone(30); - }); - - it("assigns a bounded peer relay tree and authorizes only direct media edges", async () => { - const harness = await startHarness({ peerAssistedMedia: true }); - const secondRoom = harness.roomStore.createRoom(); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted( - await authenticate(host, harness.room, "host", "relay-host"), - ); - expect(hostAuth.mediaAssignment).toEqual({ - parentPeerId: null, - childPeerIds: [], - }); - expect(hostAuth).toMatchObject({ - routeRevision: 0, - routeAssignment: { - upstream: { kind: "none" }, - childPeerIds: [], - sfuPublicationGeneration: null, - }, - }); - expect("sfuStandbyUrl" in hostAuth).toBe(false); - - const firstViewer = await openClient(harness.webSocketUrl); - const firstAuth = peerAssisted( - await authenticate( - firstViewer, - harness.room, - "viewer", - "relay-viewer-a", - ), - ); - expect(firstAuth.mediaAssignment.parentPeerId).toBe(hostAuth.peerId); - expect(firstAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: hostAuth.peerId, - }); - expect((await host.inbox.next("media-assignment")).mediaAssignment).toEqual({ - parentPeerId: null, - childPeerIds: [firstAuth.peerId], - }); - - const secondViewer = await openClient(harness.webSocketUrl); - const secondAuth = peerAssisted( - await authenticate( - secondViewer, - harness.room, - "viewer", - "relay-viewer-b", - ), - ); - expect(secondAuth.mediaAssignment.parentPeerId).toBe(hostAuth.peerId); - expect((await host.inbox.next("media-assignment")).mediaAssignment).toEqual({ - parentPeerId: null, - childPeerIds: [firstAuth.peerId, secondAuth.peerId], - }); - - const thirdViewer = await openClient(harness.webSocketUrl); - const thirdAuth = peerAssisted( - await authenticate( - thirdViewer, - harness.room, - "viewer", - "relay-viewer-c", - ), - ); - expect(thirdAuth.mediaAssignment.parentPeerId).toBe(firstAuth.peerId); - expect( - (await firstViewer.inbox.next("media-assignment")).mediaAssignment, - ).toEqual({ - parentPeerId: hostAuth.peerId, - childPeerIds: [thirdAuth.peerId], - }); - - firstViewer.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: thirdAuth.peerId, - payload: { - kind: "description", - connectionId: "relay-connection", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - expect(await thirdViewer.inbox.next("signal")).toMatchObject({ - fromPeerId: firstAuth.peerId, - payload: { description: { type: "offer" } }, - }); - - thirdViewer.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: firstAuth.peerId, - payload: { - kind: "description", - connectionId: "relay-connection", - description: { type: "answer", sdp: "v=0\r\n" }, - }, - }), - ); - expect(await firstViewer.inbox.next("signal")).toMatchObject({ - fromPeerId: thirdAuth.peerId, - payload: { description: { type: "answer" } }, - }); - - thirdViewer.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: firstAuth.peerId, - payload: { - kind: "candidate", - connectionId: "relay-connection", - candidate: null, - }, - }), - ); - expect(await firstViewer.inbox.next("signal")).toMatchObject({ - fromPeerId: thirdAuth.peerId, - payload: { kind: "candidate" }, - }); - - thirdViewer.socket.send( - JSON.stringify({ - type: "restart-request", - targetPeerId: firstAuth.peerId, - connectionId: "relay-connection", - rebuild: true, - }), - ); - expect(await firstViewer.inbox.next("restart-request")).toMatchObject({ - fromPeerId: thirdAuth.peerId, - connectionId: "relay-connection", - rebuild: true, - }); - - thirdViewer.socket.send( - JSON.stringify({ - type: "restart-request", - targetPeerId: hostAuth.peerId, - connectionId: "relay-connection", - rebuild: false, - }), - ); - expect((await thirdViewer.inbox.next("error")).code).toBe("FORBIDDEN"); - - host.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: thirdAuth.peerId, - payload: { - kind: "candidate", - connectionId: "forbidden-non-edge", - candidate: null, - }, - }), - ); - expect((await host.inbox.next("error")).code).toBe("FORBIDDEN"); - - thirdViewer.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: firstAuth.peerId, - payload: { - kind: "description", - connectionId: "forbidden-direction", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - expect((await thirdViewer.inbox.next("error")).code).toBe("FORBIDDEN"); - - const foreignViewer = await openClient(harness.webSocketUrl); - const foreignAuth = peerAssisted( - await authenticate( - foreignViewer, - secondRoom, - "viewer", - "relay-foreign-viewer", - ), - ); - firstViewer.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: foreignAuth.peerId, - payload: { - kind: "candidate", - connectionId: "cross-room", - candidate: null, - }, - }), - ); - expect((await firstViewer.inbox.next("error")).code).toBe("PEER_NOT_FOUND"); - await foreignViewer.inbox.expectNone(30); - }); - - it("ignores SFU media assertions on an active peer route", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { - issueToken: async ({ peerId }) => `token-${peerId}`, - }, - }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted( - await authenticate( - host, - harness.room, - "host", - "peer-media-state-host", - 1, - "peer-media-state-share", - { viewerPresence: true }, - ), - ); - const viewer = await openClient(harness.webSocketUrl); - const viewerAuth = peerAssisted( - await authenticate( - viewer, - harness.room, - "viewer", - "peer-media-state-viewer", - ), - ); - expect(viewerAuth.routeAssignment.upstream.kind).toBe("peer"); - - for (const message of [ - { - type: "route-ready", - revision: viewerAuth.routeRevision, - phase: "active", - }, - { - type: "route-media-unavailable", - revision: viewerAuth.routeRevision, - }, - ]) { - viewer.socket.send(JSON.stringify(message)); - } - - const replacementHost = await openClient(harness.webSocketUrl); - const replacementAuth = peerAssisted( - await authenticate( - replacementHost, - harness.room, - "host", - "peer-media-state-host", - 1, - "peer-media-state-share", - { viewerPresence: true }, - ), - ); - expect(replacementAuth.peerId).toBe(hostAuth.peerId); - const presence = await nextViewerPresenceMatching( - replacementHost, - (message) => - viewerPresenceEntries(message).some( - (entry) => entry.peerId === viewerAuth.peerId, - ), - ); - expect( - viewerPresenceEntries(presence).find( - (entry) => entry.peerId === viewerAuth.peerId, - ), - ).toMatchObject({ upstream: viewerAuth.routeAssignment.upstream }); - expect( - viewerPresenceEntries(presence).find( - (entry) => entry.peerId === viewerAuth.peerId, - )?.sfuMediaReady, - ).toBeUndefined(); - }); - - it("reports and revokes current SFU Viewer media readiness", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { - issueToken: async ({ peerId }) => `token-${peerId}`, - }, - }); - const active = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "sfu-media-ready", - true, - ); - const findViewer = ( - message: Extract, - ) => - viewerPresenceEntries(message).find( - (entry) => entry.peerId === active.viewerAuth.peerId, - ); - const initiallyUnready = await nextViewerPresenceMatching( - active.host, - (message) => findViewer(message)?.upstream.kind === "sfu", - ); - expect(findViewer(initiallyUnready)?.sfuMediaReady).toBeUndefined(); - - active.viewer.socket.send( - JSON.stringify({ - type: "route-ready", - revision: active.revision - 1, - phase: "active", - }), - ); - const replacementHost = await openClient(harness.webSocketUrl); - await authenticate( - replacementHost, - harness.room, - "host", - "sfu-media-ready-host", - 1, - "sfu-media-ready-share", - { viewerPresence: true }, - ); - const afterStaleReady = await nextViewerPresenceMatching( - replacementHost, - (message) => findViewer(message)?.upstream.kind === "sfu", - ); - expect(findViewer(afterStaleReady)?.sfuMediaReady).toBeUndefined(); - - active.viewer.socket.send( - JSON.stringify({ - type: "route-ready", - revision: active.revision, - phase: "active", - }), - ); - const ready = await nextViewerPresenceMatching( - replacementHost, - (message) => findViewer(message)?.sfuMediaReady === true, - ); - expect(findViewer(ready)).toMatchObject({ - upstream: { kind: "sfu" }, - sfuMediaReady: true, - }); - - active.viewer.socket.send( - JSON.stringify({ - type: "route-media-unavailable", - revision: active.revision - 1, - }), - ); - const replayHost = await openClient(harness.webSocketUrl); - await authenticate( - replayHost, - harness.room, - "host", - "sfu-media-ready-host", - 1, - "sfu-media-ready-share", - { viewerPresence: true }, - ); - const replayed = await nextViewerPresenceMatching( - replayHost, - (message) => findViewer(message)?.sfuMediaReady === true, - ); - expect(findViewer(replayed)?.sfuMediaReady).toBe(true); - - active.viewer.socket.send( - JSON.stringify({ - type: "route-media-unavailable", - revision: active.revision, - }), - ); - const unavailable = await nextViewerPresenceMatching( - replayHost, - (message) => { - const viewer = findViewer(message); - return viewer?.upstream.kind === "sfu" && !viewer.sfuMediaReady; - }, - ); - expect(findViewer(unavailable)?.sfuMediaReady).toBeUndefined(); - - active.viewer.socket.send( - JSON.stringify({ - type: "route-ready", - revision: active.revision, - phase: "active", - }), - ); - await nextViewerPresenceMatching( - replayHost, - (message) => findViewer(message)?.sfuMediaReady === true, - ); - - const replacementViewer = await openClient(harness.webSocketUrl); - const replacementViewerAuth = peerAssisted( - await authenticate( - replacementViewer, - harness.room, - "viewer", - "sfu-media-ready-viewer", - ), - ); - expect(replacementViewerAuth).toMatchObject({ - peerId: active.viewerAuth.peerId, - routeRevision: active.revision, - routeAssignment: { upstream: { kind: "sfu" } }, - }); - const afterViewerSessionChange = await nextViewerPresenceMatching( - replayHost, - (message) => { - const viewer = findViewer(message); - return viewer?.upstream.kind === "sfu" && !viewer.sfuMediaReady; - }, - ); - expect(findViewer(afterViewerSessionChange)?.sfuMediaReady).toBeUndefined(); - - replacementViewer.socket.send( - JSON.stringify({ - type: "route-ready", - revision: replacementViewerAuth.routeRevision, - phase: "active", - }), - ); - await nextViewerPresenceMatching( - replayHost, - (message) => findViewer(message)?.sfuMediaReady === true, - ); - - await closeClient(replacementViewer); - await nextViewerPresenceMatching( - replayHost, - (message) => findViewer(message) === undefined, - ); - }); - - it("reserves a Host publication slot when deployment tightens its edge limit to one", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { - async issueToken({ peerId }) { - return `token-${peerId}`; - }, - }, - endpointMediaCopyCapacity: 1, - }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted( - await authenticate(host, harness.room, "host", "limit-one-host"), - ); - const branchRoot = await openClient(harness.webSocketUrl); - const branchRootAuth = peerAssisted( - await authenticate( - branchRoot, - harness.room, - "viewer", - "limit-one-branch-root", - 1, - ), - ); - expect(branchRootAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: hostAuth.peerId, - }); - const failedViewer = await openClient(harness.webSocketUrl); - const failedAuth = peerAssisted( - await authenticate( - failedViewer, - harness.room, - "viewer", - "limit-one-failed-viewer", - 0, - ), - ); - expect(failedAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: branchRootAuth.peerId, - }); - - await sendTestOffer( - host, - branchRoot, - branchRootAuth.peerId, - "limit_one_host_branch_connection", - ); - await sendTestOffer( - branchRoot, - failedViewer, - failedAuth.peerId, - "limit_one_deep_connection", - ); - failedViewer.socket.send(JSON.stringify({ - type: "route-failed", - revision: failedAuth.routeRevision, - phase: "active", - connectionId: "limit_one_deep_connection", - })); - - const hostPrepare = await nextPreparedRoute(host); - const branchPrepare = await nextPreparedRoute(branchRoot); - const failedPrepare = await nextPreparedRoute(failedViewer); - expect(branchPrepare.revision).toBe(hostPrepare.revision); - expect(failedPrepare.revision).toBe(hostPrepare.revision); - expect(hostPrepare.assignment).toMatchObject({ - childPeerIds: [], - sfuPublicationGeneration: expect.any(String), - }); - expect(branchPrepare.assignment.upstream).toEqual({ kind: "sfu" }); - expect(failedPrepare.assignment.upstream).toEqual({ kind: "sfu" }); - }); - - it("rejects a fourth Host copy before issuing an SFU token at capacity three", async () => { - let issuedSfuTokens = 0; - const harness = await startSfuHarness({ - tokenIssuer: { - async issueToken({ peerId }) { - issuedSfuTokens += 1; - return `token-${peerId}`; - }, - }, - endpointMediaCopyCapacity: 3, - }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted( - await authenticate(host, harness.room, "host", "capacity-three-host"), - ); - const viewers: Array<{ - client: TestClient; - auth: ReturnType; - }> = []; - for (let index = 0; index < 3; index += 1) { - const client = await openClient(harness.webSocketUrl); - const auth = peerAssisted( - await authenticate( - client, - harness.room, - "viewer", - `capacity-three-viewer-${index}`, - 0, - ), - ); - expect(auth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: hostAuth.peerId, - }); - viewers.push({ client, auth }); - } - - const failed = viewers[2]!; - const connectionId = "capacity_three_third_edge"; - await sendTestOffer( - host, - failed.client, - failed.auth.peerId, - connectionId, - ); - failed.client.socket.send(JSON.stringify({ - type: "route-failed", - revision: failed.auth.routeRevision, - phase: "active", - connectionId, - })); - - expect(await failed.client.inbox.next("error")).toMatchObject({ - code: "PEER_NOT_FOUND", - message: "Endpoint media copy transition capacity is exhausted", - }); - expect(issuedSfuTokens).toBe(0); - await expect(failed.client.inbox.next("sfu-config", 40)).rejects.toThrow( - "Timed out", - ); - await expect(failed.client.inbox.next("route-update", 40)).rejects.toThrow( - "Timed out", - ); - }); - - it("retains a zero-child SFU root across commit and reauthentication", async () => { - const issued: Array< - Parameters[0] - > = []; - const harness = await startSfuHarness({ - tokenIssuer: { - async issueToken(request) { - issued.push(request); - return `token-${request.peerId}`; - }, - }, - }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted( - await authenticate(host, harness.room, "host", "hybrid-host"), - ); - expect(hostAuth.sfuStandbyUrl).toBe("wss://sfu.example.test"); - const firstViewer = await openClient(harness.webSocketUrl); - const firstAuth = peerAssisted( - await authenticate( - firstViewer, - harness.room, - "viewer", - "hybrid-viewer-a", - ), - ); - expect(firstAuth.sfuStandbyUrl).toBe("wss://sfu.example.test"); - const secondViewer = await openClient(harness.webSocketUrl); - const secondAuth = peerAssisted( - await authenticate( - secondViewer, - harness.room, - "viewer", - "hybrid-viewer-b", - ), - ); - const firstBeforeFailure = await nextActiveRouteAfter( - firstViewer, - firstAuth.routeRevision, - ); - - host.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: firstAuth.peerId, - payload: { - kind: "description", - connectionId: "hybrid-direct-connection", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await firstViewer.inbox.next("signal"); - firstViewer.socket.send( - JSON.stringify({ - type: "route-failed", - revision: firstBeforeFailure.revision, - phase: "active", - connectionId: "hybrid-direct-connection", - }), - ); - - const firstReparented = await nextActiveRouteAfter( - firstViewer, - firstBeforeFailure.revision, - ); - expect(firstReparented.assignment.upstream).toEqual({ - kind: "peer", - peerId: secondAuth.peerId, - }); - const hostAfterReparent = await nextActiveRouteAfter( - host, - firstBeforeFailure.revision, - ); - expect(hostAfterReparent.assignment).toMatchObject({ - childPeerIds: [secondAuth.peerId], - sfuPublicationGeneration: null, - }); - - secondViewer.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: firstAuth.peerId, - payload: { - kind: "description", - connectionId: "hybrid-deep-connection", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await firstViewer.inbox.next("signal"); - firstViewer.socket.send( - JSON.stringify({ - type: "route-failed", - revision: firstReparented.revision, - phase: "active", - connectionId: "hybrid-deep-connection", - }), - ); - - const hostPrepare = await nextPreparedRoute(host); - const rootPrepare = await nextPreparedRoute(firstViewer); - expect(rootPrepare.revision).toBe(hostPrepare.revision); - expect(hostPrepare.assignment).toMatchObject({ - upstream: { kind: "none" }, - childPeerIds: [secondAuth.peerId], - }); - expect(rootPrepare.assignment).toMatchObject({ - upstream: { kind: "sfu" }, - childPeerIds: [], - }); - expect(await host.inbox.next("sfu-config")).toMatchObject({ - revision: hostPrepare.revision, - url: "wss://sfu.example.test", - }); - expect(await firstViewer.inbox.next("sfu-config")).toMatchObject({ - revision: hostPrepare.revision, - }); - await expect(secondViewer.inbox.next("sfu-config", 30)).rejects.toThrow( - "Timed out", - ); - - secondViewer.socket.send( - JSON.stringify({ - type: "route-ready", - revision: hostPrepare.revision, - phase: "prepare", - }), - ); - expect((await secondViewer.inbox.next("error")).code).toBe("FORBIDDEN"); - host.socket.send( - JSON.stringify({ - type: "route-ready", - revision: hostPrepare.revision, - phase: "prepare", - }), - ); - firstViewer.socket.send( - JSON.stringify({ - type: "route-ready", - revision: hostPrepare.revision, - phase: "prepare", - }), - ); - - const hostActive = await nextActiveRouteAfter( - host, - hostPrepare.revision - 1, - ); - const rootActive = await nextActiveRouteAfter( - firstViewer, - hostPrepare.revision - 1, - ); - expect(hostActive.revision).toBe(hostPrepare.revision); - expect(hostActive.assignment.childPeerIds).toEqual([secondAuth.peerId]); - expect(hostActive.assignment.sfuPublicationGeneration).toBeTruthy(); - expect(rootActive.assignment.upstream).toEqual({ kind: "sfu" }); - expect(rootActive.assignment.childPeerIds).toEqual([]); - expect(new Set(issued.map(({ peerId }) => peerId))).toEqual( - new Set([hostAuth.peerId, firstAuth.peerId]), - ); - expect(new Set(issued.map(({ publicationGeneration }) => publicationGeneration)).size).toBe(1); - - host.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: firstAuth.peerId, - payload: { - kind: "candidate", - connectionId: "no-longer-direct", - candidate: null, - }, - }), - ); - expect((await host.inbox.next("error")).code).toBe("FORBIDDEN"); - - await closeClient(firstViewer); - const reconnectedRoot = await openClient(harness.webSocketUrl); - const reconnectedRootAuth = peerAssisted( - await authenticate( - reconnectedRoot, - harness.room, - "viewer", - "hybrid-viewer-a", - ), - ); - expect(reconnectedRootAuth).toMatchObject({ - peerId: firstAuth.peerId, - routeRevision: hostPrepare.revision, - routeAssignment: { - upstream: { kind: "sfu" }, - childPeerIds: [], - }, - mediaAssignment: { parentPeerId: null }, - }); - expect(await reconnectedRoot.inbox.next("sfu-config")).toMatchObject({ - revision: hostPrepare.revision, - }); - - const issuedBeforeRefresh = issued.length; - for (let requestIndex = 0; requestIndex < 2; requestIndex += 1) { - reconnectedRoot.socket.send( - JSON.stringify({ - type: "refresh-sfu", - revision: hostPrepare.revision, - }), - ); - } - expect(await reconnectedRoot.inbox.next("sfu-config")).toMatchObject({ - revision: hostPrepare.revision, - }); - expect((await reconnectedRoot.inbox.next("error")).code).toBe("FORBIDDEN"); - expect(issued).toHaveLength(issuedBeforeRefresh + 1); - const issuedBeforeFailback = issued.length; - reconnectedRoot.socket.send( - JSON.stringify({ - type: "route-failed", - revision: hostPrepare.revision, - phase: "active", - connectionId: null, - }), - ); - const peerBaseline = await nextActiveRouteAfter( - reconnectedRoot, - hostPrepare.revision, - ); - const hostPeerBaseline = await nextActiveRouteAfter( - host, - hostPrepare.revision, - ); - expect(peerBaseline.assignment.upstream).toEqual({ - kind: "peer", - peerId: secondAuth.peerId, - }); - expect(hostPeerBaseline.assignment).toMatchObject({ - childPeerIds: [secondAuth.peerId], - sfuPublicationGeneration: null, - }); - - secondViewer.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: firstAuth.peerId, - payload: { - kind: "description", - connectionId: "after-sfu-root-connection", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await reconnectedRoot.inbox.next("signal"); - reconnectedRoot.socket.send( - JSON.stringify({ - type: "route-failed", - revision: peerBaseline.revision, - phase: "active", - connectionId: "after-sfu-root-connection", - }), - ); - const directAgain = await nextActiveRouteAfter( - reconnectedRoot, - peerBaseline.revision, - ); - expect(directAgain.assignment.upstream).toEqual({ - kind: "peer", - peerId: hostAuth.peerId, - }); - host.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: firstAuth.peerId, - payload: { - kind: "description", - connectionId: "after-sfu-host-connection", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await reconnectedRoot.inbox.next("signal"); - reconnectedRoot.socket.send( - JSON.stringify({ - type: "route-failed", - revision: directAgain.revision, - phase: "active", - connectionId: "after-sfu-host-connection", - }), - ); - expect((await reconnectedRoot.inbox.next("error")).code).toBe("PEER_NOT_FOUND"); - expect(issued).toHaveLength(issuedBeforeFailback); - - host.socket.send(JSON.stringify({ type: "stop-sharing" })); - await reconnectedRoot.inbox.next("sharing-stopped"); - const resumedHost = await openClient(harness.webSocketUrl); - const resumedHostAuth = peerAssisted( - await authenticate(resumedHost, harness.room, "host", "hybrid-host"), - ); - expect(resumedHostAuth.routeAssignment).toMatchObject({ - upstream: { kind: "none" }, - sfuPublicationGeneration: null, - }); - expect(resumedHostAuth.routeAssignment.childPeerIds).toHaveLength(2); - - const resumedFirstRoute = await nextActiveRouteRevision( - reconnectedRoot, - resumedHostAuth.routeRevision, - ); - resumedHost.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: firstAuth.peerId, - payload: { - kind: "description", - connectionId: "resumed-direct-connection", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await reconnectedRoot.inbox.next("signal"); - reconnectedRoot.socket.send( - JSON.stringify({ - type: "route-failed", - revision: resumedFirstRoute.revision, - phase: "active", - connectionId: "resumed-direct-connection", - }), - ); - const resumedReparent = await nextActiveRouteAfter( - reconnectedRoot, - resumedFirstRoute.revision, - ); - secondViewer.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: firstAuth.peerId, - payload: { - kind: "description", - connectionId: "resumed-deep-connection", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await reconnectedRoot.inbox.next("signal"); - reconnectedRoot.socket.send( - JSON.stringify({ - type: "route-failed", - revision: resumedReparent.revision, - phase: "active", - connectionId: "resumed-deep-connection", - }), - ); - const resumedPrepare = await nextPreparedRoute(resumedHost); - expect(await resumedHost.inbox.next("sfu-config")).toMatchObject({ - revision: resumedPrepare.revision, - }); - expect(issued.length).toBeGreaterThan(issuedBeforeFailback); - }); - - it("falls through to selected TURN when SFU prepare is unavailable", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken() { throw new Error("offline"); } }, - selectedEdgeTurn: true, - }); - const direct = await failSingleViewerDirect( - harness.webSocketUrl, - harness.room, - "selected-unavailable", - ); - const [viewerGrant] = await Promise.all([ - direct.viewer.inbox.next("selected-edge-turn"), - direct.host.inbox.next("selected-edge-turn"), - ]); - expect(viewerGrant.oldConnectionId).toBe(direct.oldConnectionId); - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 0, egress: 0 }); - }); - - it("commits one healthy SFU root probe only after the peer edge is proven", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken({ peerId }) { return `token-${peerId}`; } }, - }); - const active = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "healthy-commit", - ); - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 1, egress: 1 }); - - requestHealthyReselection(active.viewer, active.revision - 1); - await expect(active.viewer.inbox.next("route-update", 40)).rejects.toThrow("Timed out"); - requestHealthyReselection(active.viewer, active.revision); - const [rootPrepare, hostPrepare] = await Promise.all([ - nextPreparedRoute(active.viewer), - nextPreparedRoute(active.host), - ]); - expect(rootPrepare.assignment.upstream).toEqual({ - kind: "peer", - peerId: active.hostAuth.peerId, - }); - expect(hostPrepare.assignment).toMatchObject({ - childPeerIds: [active.viewerAuth.peerId], - sfuPublicationGeneration: null, - }); - - requestHealthyReselection(active.viewer, active.revision); - active.viewer.socket.send(JSON.stringify({ - type: "route-ready", - revision: rootPrepare.revision, - phase: "prepare", - })); - await expect(active.viewer.inbox.next("route-update", 40)).rejects.toThrow("Timed out"); - - const connectionId = "healthy-commit-probe"; - active.host.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: active.viewerAuth.peerId, - payload: { - kind: "description", - connectionId, - description: { type: "offer", sdp: "v=0\r\n" }, - }, - })); - expect(await active.viewer.inbox.next("signal")).toMatchObject({ - fromPeerId: active.hostAuth.peerId, - payload: { connectionId }, - }); - active.viewer.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: active.hostAuth.peerId, - payload: { - kind: "description", - connectionId: `${connectionId}-stale`, - description: { type: "answer", sdp: "v=0\r\n" }, - }, - })); - expect((await active.viewer.inbox.next("error")).code).toBe("FORBIDDEN"); - active.viewer.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: active.hostAuth.peerId, - payload: { - kind: "description", - connectionId, - description: { type: "answer", sdp: "v=0\r\n" }, - }, - })); - await active.host.inbox.next("signal"); - active.viewer.socket.send(JSON.stringify({ - type: "route-ready", - revision: rootPrepare.revision, - phase: "prepare", - })); - const [rootActive, hostActive] = await Promise.all([ - nextActiveRouteRevision(active.viewer, rootPrepare.revision), - nextActiveRouteRevision(active.host, hostPrepare.revision), - ]); - expect(rootActive.assignment.upstream).toEqual({ - kind: "peer", - peerId: active.hostAuth.peerId, - }); - expect(hostActive.assignment.sfuPublicationGeneration).toBeNull(); - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 0, egress: 0 }); - - active.viewer.socket.send(JSON.stringify({ - type: "route-failed", - revision: rootActive.revision, - phase: "active", - connectionId, - })); - const [rootRecovery, hostRecovery] = await Promise.all([ - nextPreparedRoute(active.viewer), - nextPreparedRoute(active.host), - ]); - expect(rootRecovery.assignment.upstream).toEqual({ kind: "sfu" }); - expect(hostRecovery.assignment.sfuPublicationGeneration).toBeTruthy(); - }); - - it("rolls back and reasserts one healthy ready after room cooldown", async () => { - let now = 50_000; - const reselections = vi.spyOn( - HybridMediaRouter.prototype, - "handleHealthySfuReselection", - ); - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken({ peerId }) { return `token-${peerId}`; } }, - prepareTimeoutMs: 200, - now: () => now, - }); - const active = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "healthy-rollback", - ); - requestHealthyReselection(active.viewer, active.revision); - const [rootPrepare] = await Promise.all([ - nextPreparedRoute(active.viewer), - nextPreparedRoute(active.host), - ]); - const connectionId = "healthy-rollback-probe"; - active.host.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: active.viewerAuth.peerId, - payload: { - kind: "description", - connectionId, - description: { type: "offer", sdp: "v=0\r\n" }, - }, - })); - await active.viewer.inbox.next("signal"); - active.viewer.socket.send(JSON.stringify({ - type: "route-failed", - revision: rootPrepare.revision, - phase: "prepare", - connectionId, - })); - const [rootRollback, hostRollback] = await Promise.all([ - nextActiveRouteAfter(active.viewer, rootPrepare.revision), - nextActiveRouteAfter(active.host, rootPrepare.revision), - ]); - expect(rootRollback.assignment.upstream).toEqual({ kind: "sfu" }); - expect(hostRollback.assignment.sfuPublicationGeneration).toBeTruthy(); - - const router = reselections.mock.instances.at(-1) as - | HybridMediaRouter - | undefined; - const currentViewer = harness.roomStore.getConnectedViewer( - harness.room.roomId, - active.viewerAuth.peerId, - ); - expect(router).toBeDefined(); - expect(currentViewer).toBeDefined(); - const deferredState = (router as unknown as { - deferredHealthySfuReselections: Map< - string, - { timer: NodeJS.Timeout | null } - >; - }).deferredHealthySfuReselections; - vi.useFakeTimers(); - try { - router!.handleHealthySfuReselection( - { - roomId: harness.room.roomId, - role: "viewer", - peerId: active.viewerAuth.peerId, - sessionId: currentViewer!.sessionId, - }, - rootRollback.revision, - ); - const deferred = deferredState.get(harness.room.roomId); - expect(deferred?.timer?.hasRef()).toBe(false); - - now += 30_001; - await vi.advanceTimersByTimeAsync(30_000); - expect(await active.viewer.inbox.next("route-update")).toMatchObject({ - revision: rootRollback.revision, - phase: "active", - assignment: { upstream: { kind: "sfu" } }, - }); - } finally { - vi.useRealTimers(); - reselections.mockRestore(); - } - - // The real Viewer re-proves two fresh stats windows before this second ready. - requestHealthyReselection(active.viewer, rootRollback.revision); - const retried = await nextPreparedRoute(active.viewer); - expect(retried.assignment.upstream).toEqual({ - kind: "peer", - peerId: active.hostAuth.peerId, - }); - const timedOut = await nextActiveRouteAfter(active.viewer, retried.revision); - expect(timedOut.assignment.upstream).toEqual({ kind: "sfu" }); - - router!.handleHealthySfuReselection( - { - roomId: harness.room.roomId, - role: "viewer", - peerId: active.viewerAuth.peerId, - sessionId: currentViewer!.sessionId, - }, - timedOut.revision, - ); - expect(deferredState.has(harness.room.roomId)).toBe(true); - active.host.socket.send(JSON.stringify({ type: "stop-sharing" })); - await active.viewer.inbox.next("sharing-stopped"); - expect(deferredState.has(harness.room.roomId)).toBe(false); - }); - - it("does not swallow active SFU loss while a healthy probe is pending", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken({ peerId }) { return `token-${peerId}`; } }, - }); - const active = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "healthy-active-loss", - ); - requestHealthyReselection(active.viewer, active.revision); - const pending = await nextPreparedRoute(active.viewer); - await nextPreparedRoute(active.host); - active.viewer.socket.send(JSON.stringify({ - type: "route-failed", - revision: active.revision, - phase: "active", - connectionId: null, - })); - let recovered = await nextActiveRouteAfter(active.viewer, pending.revision); - if (recovered.assignment.upstream.kind === "sfu") { - recovered = await nextActiveRouteAfter(active.viewer, recovered.revision); - } - expect(recovered.assignment.upstream).toEqual({ - kind: "peer", - peerId: active.hostAuth.peerId, - }); - }); - - it.each(["viewer", "host"] as const)( - "aborts a healthy probe when the active %s SFU transport sends its prepare failure", - async (failedRole) => { - const harness = await startSfuHarness({ - tokenIssuer: { - async issueToken({ peerId }) { return `token-${peerId}`; }, - }, - }); - const active = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - `healthy-${failedRole}-prepare-loss`, - ); - requestHealthyReselection(active.viewer, active.revision); - const [rootPrepare, hostPrepare] = await Promise.all([ - nextPreparedRoute(active.viewer), - nextPreparedRoute(active.host), - ]); - const failedClient = failedRole === "viewer" ? active.viewer : active.host; - const failedPrepare = failedRole === "viewer" ? rootPrepare : hostPrepare; - - failedClient.socket.send(JSON.stringify({ - type: "route-failed", - revision: active.revision, - phase: "prepare", - connectionId: null, - })); - await Promise.all([ - expect( - active.viewer.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"), - expect( - active.host.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"), - ]); - - failedClient.socket.send(JSON.stringify({ - type: "route-failed", - revision: failedPrepare.revision, - phase: "prepare", - connectionId: null, - })); - const [rootRollback, hostRollback] = await Promise.all([ - nextActiveRouteAfter(active.viewer, failedPrepare.revision), - nextActiveRouteAfter(active.host, failedPrepare.revision), - ]); - expect(rootRollback.assignment.upstream).toEqual({ kind: "sfu" }); - expect(hostRollback.assignment.sfuPublicationGeneration).toBeTruthy(); - - failedClient.socket.send(JSON.stringify({ - type: "refresh-sfu", - revision: rootRollback.revision, - })); - expect(await failedClient.inbox.next("sfu-config")).toMatchObject({ - revision: rootRollback.revision, - }); - }, - ); - - it("recovers when active selected Host ingress fails during healthy prepare", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { - async issueToken({ peerId }) { return `token-${peerId}`; }, - }, - selectedEdgeTurn: true, - }); - const active = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "healthy-selected-host-loss", - ); - active.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: active.revision, - phase: "active", - connectionId: null, - })); - const ingress = await active.host.inbox.next("selected-edge-turn"); - expect(ingress.edgeKind).toBe("host-sfu-ingress"); - active.host.socket.send(JSON.stringify({ - type: "route-ready", - revision: active.revision, - phase: "active", - })); - - requestHealthyReselection(active.viewer, active.revision); - const [rootPrepare, hostPrepare] = await Promise.all([ - nextPreparedRoute(active.viewer), - nextPreparedRoute(active.host), - ]); - active.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: active.revision, - phase: "prepare", - connectionId: ingress.newConnectionId, - })); - active.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: hostPrepare.revision, - phase: "prepare", - connectionId: `${ingress.newConnectionId}-wrong`, - })); - active.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: hostPrepare.revision, - phase: "prepare", - connectionId: null, - })); - await Promise.all([ - expect( - active.viewer.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"), - expect( - active.host.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"), - ]); - - active.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: hostPrepare.revision, - phase: "prepare", - connectionId: ingress.newConnectionId, - })); - const [rootRollback, hostRollback] = await Promise.all([ - nextActiveRouteAfter(active.viewer, rootPrepare.revision), - nextActiveRouteAfter(active.host, hostPrepare.revision), - ]); - expect(rootRollback.assignment.upstream).toEqual({ kind: "sfu" }); - expect(hostRollback.assignment.sfuPublicationGeneration).toBeTruthy(); - - active.host.socket.send(JSON.stringify({ - type: "refresh-sfu", - revision: hostRollback.revision, - })); - expect(await active.host.inbox.next("sfu-config")).toMatchObject({ - revision: hostRollback.revision, - }); - }); - - it("does not let a soft Host-parent probe swallow active SFU publication failure", async () => { - let now = 205_000; - const harness = await startSfuHarness({ - now: () => now, - endpointMediaCopyCapacity: 3, - tokenIssuer: { - async issueToken({ peerId }) { return `token-${peerId}`; }, - }, - }); - const active = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "quality-host-sfu-loss", - ); - const { child, childPrepare, current, hostPrepare, parentAuth } = - await prepareSfuHostParentQualityProbe( - harness, - active, - "quality_host_sfu_loss", - () => { now += 2_000; }, - ); - - active.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: current.revision, - phase: "prepare", - connectionId: null, - })); - await expect(child.inbox.next("route-update", 40)).rejects.toThrow( - "Timed out", - ); - - active.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: hostPrepare.revision, - phase: "prepare", - connectionId: null, - })); - const [childBaseline, firstHostRecovery] = await Promise.all([ - nextActiveRouteAfter(child, childPrepare.revision), - nextActiveRouteAfter(active.host, hostPrepare.revision), - ]); - const hostBaseline = firstHostRecovery.assignment.sfuPublicationGeneration === null - ? firstHostRecovery - : await nextActiveRouteAfter(active.host, firstHostRecovery.revision); - expect(childBaseline.assignment.upstream).toEqual({ - kind: "peer", - peerId: parentAuth.peerId, - }); - expect(hostBaseline.assignment.sfuPublicationGeneration).toBeNull(); - }); - - it("bounds selected ingress after active Host SFU failure during a soft probe", async () => { - try { - let now = 208_000; - const harness = await startSfuHarness({ - now: () => now, - endpointMediaCopyCapacity: 3, - tokenIssuer: { - async issueToken({ peerId }) { return `token-${peerId}`; }, - }, - selectedEdgeTurn: true, - }); - const active = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "quality-host-selected-timeout", - ); - const { child, childPrepare, hostPrepare, parentAuth } = - await prepareSfuHostParentQualityProbe( - harness, - active, - "quality_host_selected_timeout", - () => { now += 2_000; }, - ); - - vi.useFakeTimers(); - active.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: hostPrepare.revision, - phase: "prepare", - connectionId: null, - })); - const [childRollback, hostRollback, grant] = await Promise.all([ - nextActiveRouteAfter(child, childPrepare.revision), - nextActiveRouteAfter(active.host, hostPrepare.revision), - active.host.inbox.next("selected-edge-turn"), - ]); - expect(childRollback.assignment.upstream).toEqual({ - kind: "peer", - peerId: parentAuth.peerId, - }); - expect(hostRollback.revision).toBe(hostPrepare.revision + 1); - expect(hostRollback.assignment.sfuPublicationGeneration).toBeTruthy(); - expect(grant).toMatchObject({ - edgeKind: "host-sfu-ingress", - revision: hostRollback.revision, - }); - - await vi.advanceTimersByTimeAsync(30_000); - await expect(active.host.inbox.next("error")).resolves.toMatchObject({ - code: "PEER_NOT_FOUND", - message: "Selected SFU relay ingress timed out", - }); - await expect( - nextActiveRouteAfter(active.host, hostRollback.revision), - ).resolves.toMatchObject({ - revision: hostRollback.revision + 1, - assignment: { sfuPublicationGeneration: null }, - }); - } finally { - vi.useRealTimers(); - } - }); - - it("does not let a soft Host-parent probe swallow active selected ingress failure", async () => { - let now = 210_000; - const harness = await startSfuHarness({ - now: () => now, - endpointMediaCopyCapacity: 3, - tokenIssuer: { - async issueToken({ peerId }) { return `token-${peerId}`; }, - }, - selectedEdgeTurn: true, - }); - const active = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "quality-selected-host-loss", - ); - active.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: active.revision, - phase: "active", - connectionId: null, - })); - const ingress = await active.host.inbox.next("selected-edge-turn"); - active.host.socket.send(JSON.stringify({ - type: "route-ready", - revision: active.revision, - phase: "active", - })); - - const { child, childPrepare, current, hostPrepare } = - await prepareSfuHostParentQualityProbe( - harness, - active, - "quality_selected_host", - () => { now += 2_000; }, - ); - - active.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: current.revision, - phase: "active", - connectionId: ingress.newConnectionId, - })); - const [childBaseline, firstHostRecovery] = await Promise.all([ - nextActiveRouteAfter(child, childPrepare.revision), - nextActiveRouteAfter(active.host, hostPrepare.revision), - ]); - const hostBaseline = firstHostRecovery.assignment.sfuPublicationGeneration === null - ? firstHostRecovery - : await nextActiveRouteAfter(active.host, firstHostRecovery.revision); - expect(childBaseline.assignment.upstream.kind).toBe("peer"); - expect(hostBaseline.assignment.sfuPublicationGeneration).toBeNull(); - }); - - it("keeps ordinary intent when topology abort fills the overlap slot", async () => { - const reselections = vi.spyOn( - HybridMediaRouter.prototype, - "handleHealthySfuReselection", - ); - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken({ peerId }) { return `token-${peerId}`; } }, - }); - const active = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "healthy-no-slot", - ); - requestHealthyReselection(active.viewer, active.revision); - await nextPreparedRoute(active.viewer); - await nextPreparedRoute(active.host); - const leaf = await openClient(harness.webSocketUrl); - const leafAuth = peerAssisted(await authenticate( - leaf, - harness.room, - "viewer", - "healthy-no-slot-leaf", - 0, - )); - expect(leafAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: active.hostAuth.peerId, - }); - const rootActive = await nextActiveRouteRevision( - active.viewer, - leafAuth.routeRevision, - ); - await nextActiveRouteRevision(active.host, leafAuth.routeRevision); - const router = reselections.mock.instances.at(-1) as unknown as { - viewerRouteIntentsByRoom: Map>; - }; - const intent = router.viewerRouteIntentsByRoom - .get(harness.room.roomId) - ?.get(active.viewerAuth.peerId); - expect(intent?.sfuAttempts).toBe(1); - reselections.mockRestore(); - requestHealthyReselection(active.viewer, rootActive.revision); - await expect(active.viewer.inbox.next("route-update", 40)).rejects.toThrow("Timed out"); - await expect(active.host.inbox.next("route-update", 40)).rejects.toThrow("Timed out"); - }); - - it("keeps an SFU-fed Viewer leaf despite advertised relay capacity", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken({ peerId }) { return `token-${peerId}`; } }, - }); - const active = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "healthy-relay-budget", - ); - active.viewer.socket.send(JSON.stringify({ - type: "relay-capacity", - downstreamEdges: 3, - })); - await new Promise((resolve) => setTimeout(resolve, 0)); - const hostFiller = await openClient(harness.webSocketUrl); - const hostFillerAuth = peerAssisted(await authenticate( - hostFiller, - harness.room, - "viewer", - "healthy-relay-budget-host-filler", - 0, - )); - expect(hostFillerAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: active.hostAuth.peerId, - }); - const rootChild = await openClient(harness.webSocketUrl); - const rootChildAuth = peerAssisted(await authenticate( - rootChild, - harness.room, - "viewer", - "healthy-relay-budget-root-child", - 0, - )); - expect(rootChildAuth.routeAssignment.upstream).toEqual({ kind: "none" }); - const rootActive = await nextActiveRouteRevision( - active.viewer, - rootChildAuth.routeRevision, - ); - expect(rootActive.assignment.childPeerIds).toEqual([]); - }); - - it("cleans a healthy probe across Viewer session replacement and stop", async () => { - let now = 90_000; - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken({ peerId }) { return `token-${peerId}`; } }, - now: () => now, - }); - const active = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "healthy-session", - ); - requestHealthyReselection(active.viewer, active.revision); - const [oldPrepare] = await Promise.all([ - nextPreparedRoute(active.viewer), - nextPreparedRoute(active.host), - ]); - const oldClosed = new Promise((resolve) => - active.viewer.socket.once("close", () => resolve()), - ); - const replacement = await openClient(harness.webSocketUrl); - const replacementAuth = peerAssisted(await authenticate( - replacement, - harness.room, - "viewer", - "healthy-session-viewer", - )); - await oldClosed; - expect(replacementAuth.routeRevision).toBeGreaterThan(oldPrepare.revision); - expect(replacementAuth.routeAssignment.upstream).toEqual({ kind: "sfu" }); - const hostRollback = await nextActiveRouteAfter(active.host, oldPrepare.revision); - expect(hostRollback.assignment.sfuPublicationGeneration).toBeTruthy(); - - replacement.socket.send(JSON.stringify({ - type: "route-ready", - revision: oldPrepare.revision, - phase: "prepare", - })); - requestHealthyReselection(replacement, replacementAuth.routeRevision); - await expect(replacement.inbox.next("route-update", 40)).rejects.toThrow("Timed out"); - now += 30_001; - requestHealthyReselection(replacement, replacementAuth.routeRevision); - const replacementPrepare = await nextPreparedRoute(replacement); - await nextPreparedRoute(active.host); - - active.host.socket.send(JSON.stringify({ - type: "stop-sharing", - shareGeneration: "healthy-session-share", - })); - await replacement.inbox.next("sharing-stopped"); - replacement.socket.send(JSON.stringify({ - type: "route-ready", - revision: replacementPrepare.revision, - phase: "prepare", - })); - await expect(replacement.inbox.next("route-update", 40)).rejects.toThrow("Timed out"); - }); - - it("keeps selected TURN exclusive with healthy probes until failure", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken({ peerId }) { return `token-${peerId}`; } }, - selectedEdgeTurn: true, - }); - const direct = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "selected-host-parent", - ); - expect(direct.hostAuth.iceConfig.iceServers).toEqual([]); - direct.viewer.socket.send(JSON.stringify({ - type: "refresh-sfu", - revision: direct.revision, - })); - await direct.viewer.inbox.next("sfu-config"); - direct.viewer.socket.send(JSON.stringify({ - type: "route-failed", - revision: direct.revision, - phase: "active", - connectionId: null, - })); - const [viewerGrant, hostGrant] = await Promise.all([ - direct.viewer.inbox.next("selected-edge-turn"), - direct.host.inbox.next("selected-edge-turn"), - ]); - expect(hostGrant).toEqual(viewerGrant); - expect(viewerGrant).toMatchObject({ - revision: direct.revision, - parentPeerId: direct.hostAuth.peerId, - viewerPeerId: direct.viewerAuth.peerId, - oldConnectionId: direct.oldConnectionId, - iceServer: { urls: ["turn:turn.example.test:3478?transport=udp"] }, - }); - expect(viewerGrant.newConnectionId).not.toBe(direct.oldConnectionId); - requestHealthyReselection(direct.viewer, direct.revision); - await expect( - direct.viewer.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"); - direct.host.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: direct.viewerAuth.peerId, - payload: { - kind: "description", - connectionId: viewerGrant.newConnectionId, - description: { type: "offer", sdp: "v=0\r\n" }, - }, - })); - expect(await direct.viewer.inbox.next("signal")).toMatchObject({ - fromPeerId: direct.hostAuth.peerId, - payload: { connectionId: viewerGrant.newConnectionId }, - }); - direct.viewer.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: direct.hostAuth.peerId, - payload: { - kind: "description", - connectionId: viewerGrant.newConnectionId, - description: { type: "answer", sdp: "v=0\r\n" }, - }, - })); - await direct.host.inbox.next("signal"); - requestHealthyReselection(direct.viewer, direct.revision); - await expect( - direct.viewer.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"); - direct.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: direct.revision, - phase: "active", - connectionId: viewerGrant.newConnectionId, - })); - expect((await direct.host.inbox.next("error")).code).toBe("PEER_NOT_FOUND"); - await expect( - direct.viewer.inbox.next("selected-edge-turn", 40), - ).rejects.toThrow("Timed out"); - }); - - it("keeps a healthy probe exclusive when an SFU refresh token fails", async () => { - let failRefresh = false; - let failedRefreshes = 0; - const harness = await startSfuHarness({ - tokenIssuer: { - async issueToken({ peerId }) { - if (failRefresh) { - failedRefreshes += 1; - throw new Error("refresh failed"); - } - return `token-${peerId}`; - }, - }, - selectedEdgeTurn: true, - }); - const active = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "healthy-refresh-failure", - ); - requestHealthyReselection(active.viewer, active.revision); - await Promise.all([ - nextPreparedRoute(active.viewer), - nextPreparedRoute(active.host), - ]); - - failRefresh = true; - active.viewer.socket.send(JSON.stringify({ - type: "refresh-sfu", - revision: active.revision, - })); - await vi.waitFor(() => expect(failedRefreshes).toBe(1)); - await Promise.all([ - expect( - active.viewer.inbox.next("selected-edge-turn", 40), - ).rejects.toThrow("Timed out"), - expect( - active.host.inbox.next("selected-edge-turn", 40), - ).rejects.toThrow("Timed out"), - expect( - active.viewer.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"), - expect( - active.host.inbox.next("route-update", 40), - ).rejects.toThrow("Timed out"), - ]); - }); - - it("admits independent pending peer-selected TURN edges concurrently", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken() { throw new Error("offline"); } }, - selectedEdgeTurn: true, - turnAllocationCapacity: 2, - }); - const peers = await prepareTwoSelectedPeerEdgeCandidates( - harness.webSocketUrl, - harness.room, - "selected-room-pending-cap", - ); - - const firstGrant = await startPeerSelectedCandidate( - peers.firstViewer, - peers.firstParent, - peers.revision, - peers.firstConnectionId, - ); - - failPeerEdgeCandidate( - peers.secondViewer, - firstGrant.revision, - peers.secondConnectionId, - ); - const [secondViewerGrant, secondParentGrant] = await Promise.all([ - peers.secondViewer.inbox.next("selected-edge-turn"), - peers.secondParent.inbox.next("selected-edge-turn"), - ]); - expect(secondViewerGrant).toMatchObject({ - edgeKind: "peer-selected", - viewerPeerId: peers.secondAuth.peerId, - parentPeerId: peers.secondParentAuth.peerId, - }); - expect(secondParentGrant.newConnectionId).toBe( - secondViewerGrant.newConnectionId, - ); - expect(secondViewerGrant.newConnectionId).not.toBe( - firstGrant.newConnectionId, - ); - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 2 }); - }); - - it.each([1, 2, 3] as const)( - "admits %i same-parent selected edges without changing endpoint copy count", - async (endpointMediaCopyCapacity) => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken() { throw new Error("offline"); } }, - selectedEdgeTurn: true, - endpointMediaCopyCapacity, - turnAllocationCapacity: endpointMediaCopyCapacity, - }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted(await authenticate( - host, - harness.room, - "host", - `selected-same-parent-cap-${endpointMediaCopyCapacity}-host`, - endpointMediaCopyCapacity, - `selected-same-parent-cap-${endpointMediaCopyCapacity}-share`, - )); - const viewers: Array<{ - client: TestClient; - peerId: string; - connectionId: string; - }> = []; - let revision = hostAuth.routeRevision; - for (let index = 0; index < endpointMediaCopyCapacity; index += 1) { - const client = await openClient(harness.webSocketUrl); - const authenticated = peerAssisted(await authenticate( - client, - harness.room, - "viewer", - `selected-same-parent-cap-${endpointMediaCopyCapacity}-viewer-${index}`, - 0, - )); - expect(authenticated.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: hostAuth.peerId, - }); - revision = authenticated.routeRevision; - viewers.push({ - client, - peerId: authenticated.peerId, - connectionId: - `selected-same-parent-cap-${endpointMediaCopyCapacity}-edge-${index}`, - }); - } - - for (const [index, viewer] of viewers.entries()) { - await sendTestOffer( - host, - viewer.client, - viewer.peerId, - viewer.connectionId, - ); - const grant = await startPeerSelectedCandidate( - viewer.client, - host, - revision, - viewer.connectionId, - ).catch((error: unknown) => { - throw new Error( - `selected edge ${index + 1} of ${endpointMediaCopyCapacity} was not admitted`, - { cause: error }, - ); - }); - expect(grant.parentPeerId).toBe(hostAuth.peerId); - revision = grant.revision; - } - expect(harness.turnAdmission?.usage()).toEqual({ - allocations: endpointMediaCopyCapacity, - }); - - const overflow = await openClient(harness.webSocketUrl); - const overflowAuth = peerAssisted(await authenticate( - overflow, - harness.room, - "viewer", - `selected-same-parent-cap-${endpointMediaCopyCapacity}-overflow`, - 0, - )); - expect(overflowAuth.routeAssignment.upstream).toEqual({ kind: "none" }); - expect(harness.turnAdmission?.usage()).toEqual({ - allocations: endpointMediaCopyCapacity, - }); - }, - ); - - it("releases every selected allocation when the current share stops", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken() { throw new Error("offline"); } }, - selectedEdgeTurn: true, - turnAllocationCapacity: 2, - }); - const peers = await prepareTwoSelectedPeerEdgeCandidates( - harness.webSocketUrl, - harness.room, - "selected-room-stop", - ); - - const firstGrant = await startPeerSelectedCandidate( - peers.firstViewer, - peers.firstParent, - peers.revision, - peers.firstConnectionId, - ); - await startPeerSelectedCandidate( - peers.secondViewer, - peers.secondParent, - firstGrant.revision, - peers.secondConnectionId, - ); - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 2 }); - - peers.host.socket.send(JSON.stringify({ type: "stop-sharing" })); - await Promise.all([ - peers.firstViewer.inbox.next("sharing-stopped"), - peers.secondViewer.inbox.next("sharing-stopped"), - ]); - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 0 }); - }); - - it("releases selected allocations during graceful shutdown", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken() { throw new Error("offline"); } }, - selectedEdgeTurn: true, - turnAllocationCapacity: 1, - }); - const peers = await prepareTwoSelectedPeerEdgeCandidates( - harness.webSocketUrl, - harness.room, - "selected-shutdown", - ); - await startPeerSelectedCandidate( - peers.firstViewer, - peers.firstParent, - peers.revision, - peers.firstConnectionId, - ); - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 1 }); - - await runningServer!.close(); - runningServer = undefined; - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 0 }); - }); - - it("keeps an answered peer-selected edge charged against deployment capacity", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken() { throw new Error("offline"); } }, - selectedEdgeTurn: true, - turnAllocationCapacity: 1, - }); - const peers = await prepareTwoSelectedPeerEdgeCandidates( - harness.webSocketUrl, - harness.room, - "selected-room-answered-cap", - ); - - const grant = await startPeerSelectedCandidate( - peers.firstViewer, - peers.firstParent, - peers.revision, - peers.firstConnectionId, - ); - peers.firstParent.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: peers.firstAuth.peerId, - payload: { - kind: "description", - connectionId: grant.newConnectionId, - description: { type: "offer", sdp: "v=0\r\n" }, - }, - })); - await peers.firstViewer.inbox.next("signal"); - peers.firstViewer.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: peers.firstParentAuth.peerId, - payload: { - kind: "description", - connectionId: grant.newConnectionId, - description: { type: "answer", sdp: "v=0\r\n" }, - }, - })); - await peers.firstParent.inbox.next("signal"); - - failPeerEdgeCandidate( - peers.secondViewer, - grant.revision, - peers.secondConnectionId, - ); - await expect( - peers.secondViewer.inbox.next("selected-edge-turn", 40), - ).rejects.toThrow("Timed out"); - peers.firstParent.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: peers.firstAuth.peerId, - payload: { - kind: "candidate", - connectionId: grant.newConnectionId, - candidate: null, - }, - })); - await expect(peers.firstViewer.inbox.next("signal")).resolves.toMatchObject({ - payload: { connectionId: grant.newConnectionId }, - }); - }); - - it("revokes a Host selected overlay before another child exceeds cap two", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken({ peerId }) { return `token-${peerId}`; } }, - selectedEdgeTurn: true, - }); - const active = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "selected-host-overlay-budget", - ); - const grant = await startPeerSelectedTurn( - active.viewer, - active.host, - active.revision, - ); - expect(grant.parentPeerId).toBe(active.hostAuth.peerId); - active.host.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: active.viewerAuth.peerId, - payload: { - kind: "description", - connectionId: grant.newConnectionId, - description: { type: "offer", sdp: "v=0\r\n" }, - }, - })); - await active.viewer.inbox.next("signal"); - active.viewer.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: active.hostAuth.peerId, - payload: { - kind: "description", - connectionId: grant.newConnectionId, - description: { type: "answer", sdp: "v=0\r\n" }, - }, - })); - await active.host.inbox.next("signal"); - - const sibling = await openClient(harness.webSocketUrl); - const siblingAuth = peerAssisted( - await authenticate( - sibling, - harness.room, - "viewer", - "selected-host-overlay-budget-sibling", - 0, - ), - ); - const [hostRoute, selectedViewerRoute] = await Promise.all([ - nextActiveRouteRevision(active.host, siblingAuth.routeRevision), - nextActiveRouteRevision(active.viewer, siblingAuth.routeRevision), - ]); - expect(hostRoute.assignment.childPeerIds).toContain(siblingAuth.peerId); - expect(hostRoute.assignment.sfuPublicationGeneration).not.toBeNull(); - expect( - hostRoute.assignment.childPeerIds.length + - (hostRoute.assignment.sfuPublicationGeneration ? 1 : 0), - ).toBe(2); - expect(selectedViewerRoute.assignment.upstream).toEqual({ kind: "sfu" }); - await Promise.all([ - expect( - active.host.inbox.next("selected-edge-turn", 40), - ).rejects.toThrow("Timed out"), - expect( - active.viewer.inbox.next("selected-edge-turn", 40), - ).rejects.toThrow("Timed out"), - ]); - active.host.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: active.viewerAuth.peerId, - payload: { - kind: "candidate", - connectionId: grant.newConnectionId, - candidate: null, - }, - })); - expect((await active.host.inbox.next("error")).code).toBe("FORBIDDEN"); - }); - - it("revokes a selected lease when its parent withdraws relay capacity", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken() { throw new Error("offline"); } }, - selectedEdgeTurn: true, - turnAllocationCapacity: 1, - }); - const peers = await prepareTwoSelectedPeerEdgeCandidates( - harness.webSocketUrl, - harness.room, - "selected-parent-capacity-withdrawal", - ); - const firstGrant = await startPeerSelectedCandidate( - peers.firstViewer, - peers.firstParent, - peers.revision, - peers.firstConnectionId, - ); - - peers.firstParent.socket.send(JSON.stringify({ - type: "relay-capacity", - downstreamEdges: 0, - })); - const [viewerAuthority, parentAuthority] = await Promise.all([ - nextActiveRouteRevision(peers.firstViewer, firstGrant.revision), - nextActiveRouteRevision(peers.firstParent, firstGrant.revision), - ]); - expect(viewerAuthority).toMatchObject({ - revision: firstGrant.revision, - phase: "active", - }); - expect(parentAuthority).toMatchObject({ - revision: firstGrant.revision, - phase: "active", - }); - const secondGrant = await startPeerSelectedCandidate( - peers.secondViewer, - peers.secondParent, - firstGrant.revision, - peers.secondConnectionId, - ); - expect(secondGrant).toMatchObject({ - edgeKind: "peer-selected", - parentPeerId: peers.secondParentAuth.peerId, - viewerPeerId: peers.secondAuth.peerId, - }); - }); - - it("releases one exact selected edge without affecting a newer edge", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken() { throw new Error("offline"); } }, - selectedEdgeTurn: true, - turnAllocationCapacity: 1, - }); - const peers = await prepareTwoSelectedPeerEdgeCandidates( - harness.webSocketUrl, - harness.room, - "selected-room-failure-release", - ); - - const firstGrant = await startPeerSelectedCandidate( - peers.firstViewer, - peers.firstParent, - peers.revision, - peers.firstConnectionId, - ); - peers.firstViewer.socket.send(JSON.stringify({ - type: "route-failed", - revision: firstGrant.revision, - phase: "active", - connectionId: firstGrant.newConnectionId, - })); - await peers.firstViewer.inbox.next("error"); - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 0 }); - - const secondGrant = await startPeerSelectedCandidate( - peers.secondViewer, - peers.secondParent, - firstGrant.revision, - peers.secondConnectionId, - ); - expect(secondGrant).toMatchObject({ - edgeKind: "peer-selected", - viewerPeerId: peers.secondAuth.peerId, - }); - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 1 }); - - peers.firstParent.inbox.ignore("route-update"); - peers.firstParent.inbox.ignore("media-assignment"); - peers.firstParent.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: peers.firstAuth.peerId, - payload: { - kind: "candidate", - connectionId: firstGrant.newConnectionId, - candidate: null, - }, - })); - await peers.firstParent.inbox.expectNone(40); - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 1 }); - }); - - it("carries a surviving same-parent selected edge before sibling release authority", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken() { throw new Error("offline"); } }, - selectedEdgeTurn: true, - endpointMediaCopyCapacity: 2, - turnAllocationCapacity: 2, - }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted(await authenticate( - host, - harness.room, - "host", - "selected-sibling-carry-host", - 2, - "selected-sibling-carry-share", - )); - const viewerA = await openClient(harness.webSocketUrl); - const viewerAAuth = peerAssisted(await authenticate( - viewerA, - harness.room, - "viewer", - "selected-sibling-carry-viewer-a", - 0, - )); - const viewerB = await openClient(harness.webSocketUrl); - const viewerBAuth = peerAssisted(await authenticate( - viewerB, - harness.room, - "viewer", - "selected-sibling-carry-viewer-b", - 0, - )); - expect(viewerAAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: hostAuth.peerId, - }); - expect(viewerBAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: hostAuth.peerId, - }); - await nextActiveRouteRevision(viewerA, viewerBAuth.routeRevision); - - const connectionA = "selected-sibling-carry-connection-a"; - const connectionB = "selected-sibling-carry-connection-b"; - await sendTestOffer(host, viewerA, viewerAAuth.peerId, connectionA); - await sendTestOffer(host, viewerB, viewerBAuth.peerId, connectionB); - const grantA = await startPeerSelectedCandidate( - viewerA, - host, - viewerBAuth.routeRevision, - connectionA, - ); - const grantB = await startPeerSelectedCandidate( - viewerB, - host, - grantA.revision, - connectionB, - ); - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 2 }); - - const parentAuthority: ServerMessage[] = []; - host.socket.on("message", (data) => { - const message = decodeServerMessage(data.toString()); - if ( - (message.type === "selected-edge-turn" && - message.edgeKind === "peer-selected" && - message.newConnectionId === grantB.newConnectionId) || - (message.type === "route-update" && message.phase === "active") - ) { - parentAuthority.push(message); - } - }); - viewerA.socket.send(JSON.stringify({ - type: "route-failed", - revision: grantA.revision, - phase: "active", - connectionId: grantA.newConnectionId, - })); - await expect(viewerA.inbox.next("error")).resolves.toMatchObject({ - code: "PEER_NOT_FOUND", - message: "Selected relay edge failed", - }); - await vi.waitFor(() => expect(parentAuthority).toHaveLength(2)); - expect(parentAuthority[0]).toMatchObject({ - type: "selected-edge-turn", - revision: grantB.revision, - edgeKind: "peer-selected", - viewerPeerId: viewerBAuth.peerId, - newConnectionId: grantB.newConnectionId, - }); - expect(parentAuthority[1]).toMatchObject({ - type: "route-update", - revision: grantB.revision, - phase: "active", - }); - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 1 }); - - host.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: viewerBAuth.peerId, - payload: { - kind: "candidate", - connectionId: grantB.newConnectionId, - candidate: null, - }, - })); - await expect(viewerB.inbox.next("signal")).resolves.toMatchObject({ - fromPeerId: hostAuth.peerId, - payload: { connectionId: grantB.newConnectionId }, - }); - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 1 }); - }); - - it("releases deployment capacity when the selected Viewer disconnects", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken() { throw new Error("offline"); } }, - selectedEdgeTurn: true, - turnAllocationCapacity: 1, - viewerDisconnectGraceMs: 500, - }); - const peers = await prepareTwoSelectedPeerEdgeCandidates( - harness.webSocketUrl, - harness.room, - "selected-room-disconnect-release", - ); - - const firstGrant = await startPeerSelectedCandidate( - peers.firstViewer, - peers.firstParent, - peers.revision, - peers.firstConnectionId, - ); - await closeClient(peers.firstViewer); - peers.firstParent.socket.send(JSON.stringify({ - type: "relay-capacity", - downstreamEdges: 0, - })); - - const secondCurrent = await nextActiveRouteAfter( - peers.secondViewer, - firstGrant.revision, - ); - const secondGrant = await startPeerSelectedCandidate( - peers.secondViewer, - peers.secondParent, - secondCurrent.revision, - peers.secondConnectionId, - ); - expect(secondGrant).toMatchObject({ - edgeKind: "peer-selected", - viewerPeerId: peers.secondAuth.peerId, - }); - }); - - it("releases a selected edge when its Viewer session is replaced", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken() { throw new Error("offline"); } }, - selectedEdgeTurn: true, - turnAllocationCapacity: 1, - }); - const peers = await prepareTwoSelectedPeerEdgeCandidates( - harness.webSocketUrl, - harness.room, - "selected-viewer-session-replacement", - ); - const firstGrant = await startPeerSelectedCandidate( - peers.firstViewer, - peers.firstParent, - peers.revision, - peers.firstConnectionId, - ); - - const replacement = await openClient(harness.webSocketUrl); - const replacementAuth = peerAssisted( - await authenticate( - replacement, - harness.room, - "viewer", - "selected-viewer-session-replacement-first-viewer", - 0, - ), - ); - expect(replacementAuth.peerId).toBe(peers.firstAuth.peerId); - await nextActiveRouteRevision( - peers.firstParent, - replacementAuth.routeRevision, - ); - - const secondGrant = await startPeerSelectedCandidate( - peers.secondViewer, - peers.secondParent, - replacementAuth.routeRevision, - peers.secondConnectionId, - ); - if (secondGrant.edgeKind !== "peer-selected") { - throw new Error("Expected a selected peer edge grant"); - } - expect(secondGrant.viewerPeerId).toBe(peers.secondAuth.peerId); - expect(secondGrant.newConnectionId).not.toBe(firstGrant.newConnectionId); - }); - - it("allows selected TURN for Host SFU ingress outside historical room 1", async () => { - const harness = await startSfuHarness({ - persistent: true, - tokenIssuer: { async issueToken({ peerId }) { return `token-${peerId}`; } }, - selectedEdgeTurn: true, - }); - expect(harness.room.roomId).toBe("2"); - const direct = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "selected-host-ingress-room-two", - ); - - direct.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: direct.revision, - phase: "active", - connectionId: null, - })); - await expect(direct.host.inbox.next("selected-edge-turn")).resolves.toMatchObject({ - edgeKind: "host-sfu-ingress", - revision: direct.revision, - hostPeerId: direct.hostAuth.peerId, - publicationGeneration: expect.any(String), - iceServer: { urls: ["turn:turn.example.test:3478?transport=udp"] }, - }); - }); - - it("admits Host ingress and peer-selected edges from one deployment ledger", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken({ peerId }) { return `token-${peerId}`; } }, - selectedEdgeTurn: true, - turnAllocationCapacity: 2, - }); - const direct = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "selected-independent-ingress", - ); - - direct.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: direct.revision, - phase: "active", - connectionId: null, - })); - await expect( - direct.host.inbox.next("selected-edge-turn"), - ).resolves.toMatchObject({ edgeKind: "host-sfu-ingress" }); - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 1 }); - - await startPeerSelectedTurn( - direct.viewer, - direct.host, - direct.revision, - ); - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 2 }); - }); - - it("keeps active Host ingress charged until its exact transport fails", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken({ peerId }) { return `token-${peerId}`; } }, - selectedEdgeTurn: true, - turnAllocationCapacity: 1, - }); - const active = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "selected-active-ingress-exact-release", - ); - - active.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: active.revision, - phase: "active", - connectionId: null, - })); - const grant = await active.host.inbox.next("selected-edge-turn"); - if (grant.edgeKind !== "host-sfu-ingress") { - throw new Error("expected a Host SFU ingress grant"); - } - active.host.socket.send(JSON.stringify({ - type: "route-ready", - revision: active.revision, - phase: "active", - })); - await vi.waitFor(() => - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 1 }), - ); - - active.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: active.revision, - phase: "active", - connectionId: null, - })); - await expect( - active.host.inbox.next("selected-edge-turn", 40), - ).rejects.toThrow("Timed out"); - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 1 }); - - active.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: active.revision, - phase: "active", - connectionId: grant.newConnectionId, - })); - await expect(active.host.inbox.next("error")).resolves.toMatchObject({ - code: "PEER_NOT_FOUND", - message: "Selected SFU relay ingress failed", - }); - await expect( - nextActiveRouteAfter(active.host, active.revision), - ).resolves.toMatchObject({ - assignment: { sfuPublicationGeneration: null }, - }); - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 0 }); - }); - - it("keeps active Host ingress while an answered pending replacement rolls back", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken({ peerId }) { return `token-${peerId}`; } }, - selectedEdgeTurn: true, - turnAllocationCapacity: 2, - }); - const replacement = await prepareSelectedHostIngressReplacement( - harness, - "selected-ingress-rollback", - ); - - replacement.active.host.socket.send(JSON.stringify({ - type: "route-ready", - revision: replacement.hostPrepare.revision, - phase: "prepare", - })); - await vi.waitFor(() => - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 2 }), - ); - await closeClient(replacement.candidateViewer); - const rollback = await nextActiveRouteAfter( - replacement.active.host, - replacement.hostPrepare.revision, - ); - expect(rollback.assignment.sfuPublicationGeneration).toBe( - replacement.activeGrant.publicationGeneration, - ); - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 1 }); - - replacement.active.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: rollback.revision, - phase: "active", - connectionId: replacement.candidateGrant.newConnectionId, - })); - await expect( - replacement.active.host.inbox.next("error"), - ).resolves.toMatchObject({ code: "FORBIDDEN" }); - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 1 }); - }); - - it("rolls back only the exact failed pending Host ingress candidate", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken({ peerId }) { return `token-${peerId}`; } }, - selectedEdgeTurn: true, - turnAllocationCapacity: 2, - }); - const replacement = await prepareSelectedHostIngressReplacement( - harness, - "selected-ingress-candidate-failure", - ); - - replacement.active.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: replacement.hostPrepare.revision, - phase: "prepare", - connectionId: replacement.candidateGrant.newConnectionId, - })); - await expect( - replacement.active.host.inbox.next("error"), - ).resolves.toMatchObject({ - code: "PEER_NOT_FOUND", - message: "Selected SFU relay ingress failed", - }); - const rollback = await nextActiveRouteAfter( - replacement.active.host, - replacement.hostPrepare.revision, - ); - expect(rollback).toMatchObject({ - revision: replacement.hostPrepare.revision + 1, - assignment: { - sfuPublicationGeneration: - replacement.activeGrant.publicationGeneration, - }, - }); - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 1 }); - - replacement.active.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: rollback.revision, - phase: "active", - connectionId: replacement.candidateGrant.newConnectionId, - })); - await expect( - replacement.active.host.inbox.next("error"), - ).resolves.toMatchObject({ code: "FORBIDDEN" }); - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 1 }); - }); - - it("preempts a pending replacement when its exact old active ingress fails", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken({ peerId }) { return `token-${peerId}`; } }, - selectedEdgeTurn: true, - turnAllocationCapacity: 2, - }); - const replacement = await prepareSelectedHostIngressReplacement( - harness, - "selected-ingress-active-failure", - ); - - replacement.active.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: replacement.hostPrepare.revision, - phase: "prepare", - connectionId: replacement.activeGrant.newConnectionId, - })); - await expect( - replacement.active.host.inbox.next("error"), - ).resolves.toMatchObject({ - code: "PEER_NOT_FOUND", - message: "Selected SFU relay ingress failed", - }); - const rollback = await nextActiveRouteAfter( - replacement.active.host, - replacement.hostPrepare.revision, - ); - expect(rollback.assignment.sfuPublicationGeneration).toBe( - replacement.activeGrant.publicationGeneration, - ); - const baseline = await nextActiveRouteAfter( - replacement.active.host, - rollback.revision, - ); - expect(baseline).toMatchObject({ - assignment: { sfuPublicationGeneration: null }, - }); - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 0 }); - - replacement.active.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: baseline.revision, - phase: "active", - connectionId: replacement.candidateGrant.newConnectionId, - })); - await expect( - replacement.active.host.inbox.next("error"), - ).resolves.toMatchObject({ code: "FORBIDDEN" }); - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 0 }); - }); - - it("promotes a pending Host ingress only after the route commits", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken({ peerId }) { return `token-${peerId}`; } }, - selectedEdgeTurn: true, - turnAllocationCapacity: 2, - }); - const replacement = await prepareSelectedHostIngressReplacement( - harness, - "selected-ingress-commit", - ); - - replacement.active.host.socket.send(JSON.stringify({ - type: "route-ready", - revision: replacement.hostPrepare.revision, - phase: "prepare", - })); - await vi.waitFor(() => - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 2 }), - ); - for (const client of [ - replacement.active.viewer, - replacement.candidateViewer, - ]) { - client.socket.send(JSON.stringify({ - type: "route-ready", - revision: replacement.hostPrepare.revision, - phase: "prepare", - })); - } - const committed = await nextActiveRouteRevision( - replacement.active.host, - replacement.hostPrepare.revision, - ); - expect(committed.assignment.sfuPublicationGeneration).toBe( - replacement.candidateGrant.publicationGeneration, - ); - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 1 }); - - replacement.active.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: committed.revision, - phase: "active", - connectionId: replacement.activeGrant.newConnectionId, - })); - await expect( - replacement.active.host.inbox.next("error"), - ).resolves.toMatchObject({ code: "FORBIDDEN" }); - expect(harness.turnAdmission?.usage()).toEqual({ allocations: 1 }); - }); - - it("retries an initial Host SFU prepare through one selected ingress", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken({ peerId }) { return `token-${peerId}`; } }, - selectedEdgeTurn: true, - }); - const prepared = await prepareFallbackForTwoViewers( - harness.webSocketUrl, - harness.room, - "selected-pending-ingress", - ); - const publicationGeneration = - prepared.hostPrepare.assignment.sfuPublicationGeneration; - expect(publicationGeneration).toEqual(expect.any(String)); - - prepared.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: prepared.hostPrepare.revision, - phase: "prepare", - connectionId: null, - })); - const grant = await prepared.host.inbox.next("selected-edge-turn"); - expect(grant).toMatchObject({ - edgeKind: "host-sfu-ingress", - revision: prepared.hostPrepare.revision, - hostPeerId: prepared.hostAuth.peerId, - publicationGeneration, - oldConnectionId: publicationGeneration, - newConnectionId: expect.any(String), - }); - - prepared.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: prepared.hostPrepare.revision, - phase: "prepare", - connectionId: null, - })); - await expect( - prepared.host.inbox.next("selected-edge-turn", 40), - ).rejects.toThrow("Timed out"); - - prepared.host.socket.send(JSON.stringify({ - type: "route-ready", - revision: prepared.hostPrepare.revision, - phase: "prepare", - })); - prepared.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: prepared.hostPrepare.revision, - phase: "prepare", - connectionId: null, - })); - await expect( - prepared.host.inbox.next("selected-edge-turn", 40), - ).rejects.toThrow("Timed out"); - - prepared.failedViewer.socket.send(JSON.stringify({ - type: "route-ready", - revision: prepared.rootPrepare.revision, - phase: "prepare", - })); - await expect( - nextActiveRouteRevision(prepared.host, prepared.hostPrepare.revision), - ).resolves.toMatchObject({ - assignment: { sfuPublicationGeneration: publicationGeneration }, - }); - }); - - it("keeps the default prepare open for the initial LiveKit attempt", async () => { - vi.useFakeTimers(); - try { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken({ peerId }) { return `token-${peerId}`; } }, - selectedEdgeTurn: true, - }); - const prepared = await prepareFallbackForTwoViewers( - harness.webSocketUrl, - harness.room, - "selected-pending-deadline", - ); - - await vi.advanceTimersByTimeAsync(15_000); - prepared.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: prepared.hostPrepare.revision, - phase: "prepare", - connectionId: null, - })); - - await expect( - prepared.host.inbox.next("selected-edge-turn"), - ).resolves.toMatchObject({ - edgeKind: "host-sfu-ingress", - revision: prepared.hostPrepare.revision, - }); - } finally { - vi.useRealTimers(); - } - }); - - it("replaces the initial deadline with one relay timeout and rollback", async () => { - vi.useFakeTimers(); - try { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken({ peerId }) { return `token-${peerId}`; } }, - selectedEdgeTurn: true, - prepareTimeoutMs: 5_000, - }); - const prepared = await prepareFallbackForTwoViewers( - harness.webSocketUrl, - harness.room, - "selected-pending-relay-deadline", - ); - prepared.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: prepared.hostPrepare.revision, - phase: "prepare", - connectionId: null, - })); - await prepared.host.inbox.next("selected-edge-turn"); - - await vi.advanceTimersByTimeAsync(5_000); - const noEarlyRollback = expect( - prepared.host.inbox.next("route-update", 1), - ).rejects.toThrow("Timed out"); - await vi.advanceTimersByTimeAsync(1); - await noEarlyRollback; - - await vi.advanceTimersByTimeAsync(24_999); - await expect(prepared.host.inbox.next("error")).resolves.toMatchObject({ - code: "PEER_NOT_FOUND", - message: "Selected SFU relay ingress timed out", - }); - await expect( - nextActiveRouteAfter( - prepared.host, - prepared.hostPrepare.revision, - ), - ).resolves.toMatchObject({ - revision: prepared.hostPrepare.revision + 1, - assignment: { sfuPublicationGeneration: null }, - }); - } finally { - vi.useRealTimers(); - } - }); - - it("rolls back when selected Host prepare ingress also fails", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken({ peerId }) { return `token-${peerId}`; } }, - selectedEdgeTurn: true, - }); - const prepared = await prepareFallbackForTwoViewers( - harness.webSocketUrl, - harness.room, - "selected-pending-failure", - ); - - prepared.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: prepared.hostPrepare.revision, - phase: "prepare", - connectionId: null, - })); - const grant = await prepared.host.inbox.next("selected-edge-turn"); - if (grant.edgeKind !== "host-sfu-ingress") { - throw new Error("expected a Host SFU ingress grant"); - } - prepared.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: prepared.hostPrepare.revision, - phase: "prepare", - connectionId: grant.newConnectionId, - })); - - await expect(prepared.host.inbox.next("error")).resolves.toMatchObject({ - code: "PEER_NOT_FOUND", - }); - await expect( - nextActiveRouteAfter(prepared.host, prepared.hostPrepare.revision), - ).resolves.toMatchObject({ - revision: prepared.hostPrepare.revision + 1, - assignment: { sfuPublicationGeneration: null }, - }); - prepared.host.socket.send(JSON.stringify({ - type: "route-failed", - revision: prepared.hostPrepare.revision, - phase: "prepare", - connectionId: grant.newConnectionId, - })); - await expect( - prepared.host.inbox.next("selected-edge-turn", 40), - ).rejects.toThrow("Timed out"); - }); - - it("carries a ViewerRelay selected edge across an unrelated revision", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { async issueToken({ peerId }) { return `token-${peerId}`; } }, - selectedEdgeTurn: true, - }); - const prepared = await prepareFallbackForTwoViewers( - harness.webSocketUrl, - harness.room, - "selected-viewer-parent", - "selected-viewer-parent-share", - ); - for (const client of [prepared.host, prepared.failedViewer]) { - client.socket.send(JSON.stringify({ - type: "route-ready", - revision: prepared.rootPrepare.revision, - phase: "prepare", - })); - } - const [, , parentActive] = await Promise.all([ - nextActiveRouteRevision(prepared.host, prepared.hostPrepare.revision), - nextActiveRouteRevision(prepared.failedViewer, prepared.rootPrepare.revision), - nextActiveRouteRevision(prepared.rootViewer, prepared.rootPrepare.revision), - ]); - expect(parentActive.assignment.childPeerIds).toEqual([]); - prepared.failedViewer.socket.send(JSON.stringify({ - type: "refresh-sfu", - revision: prepared.rootPrepare.revision, - })); - await prepared.failedViewer.inbox.next("sfu-config"); - prepared.failedViewer.socket.send(JSON.stringify({ - type: "route-failed", - revision: prepared.rootPrepare.revision, - phase: "active", - connectionId: null, - })); - const [childGrant, parentGrant] = await Promise.all([ - prepared.failedViewer.inbox.next("selected-edge-turn"), - prepared.rootViewer.inbox.next("selected-edge-turn"), - ]); - expect(parentGrant).toEqual(childGrant); - if (childGrant.edgeKind !== "peer-selected") { - throw new Error("expected a peer-selected grant"); - } - expect(childGrant.parentPeerId).toBe(prepared.rootAuth.peerId); - prepared.rootViewer.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: prepared.failedAuth.peerId, - payload: { - kind: "description", - connectionId: childGrant.newConnectionId, - description: { type: "offer", sdp: "v=0\r\n" }, - }, - })); - await prepared.failedViewer.inbox.next("signal"); - prepared.rootViewer.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: prepared.failedAuth.peerId, - payload: { - kind: "candidate", - connectionId: childGrant.newConnectionId, - candidate: { candidate: "parent-candidate" }, - }, - })); - await prepared.failedViewer.inbox.next("signal"); - prepared.failedViewer.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: prepared.rootAuth.peerId, - payload: { - kind: "description", - connectionId: childGrant.newConnectionId, - description: { type: "answer", sdp: "v=0\r\n" }, - }, - })); - await prepared.rootViewer.inbox.next("signal"); - prepared.failedViewer.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: prepared.rootAuth.peerId, - payload: { - kind: "candidate", - connectionId: childGrant.newConnectionId, - candidate: null, - }, - })); - await prepared.rootViewer.inbox.next("signal"); - const parentAuthorityOrder: Array<{ - type: "selected-edge-turn" | "route-update"; - revision: number; - }> = []; - prepared.rootViewer.socket.on("message", (data) => { - const message = decodeServerMessage(data.toString()); - if ( - (message.type === "selected-edge-turn" && - message.edgeKind === "peer-selected" && - message.newConnectionId === childGrant.newConnectionId) || - (message.type === "route-update" && message.phase === "active") - ) { - parentAuthorityOrder.push({ - type: message.type, - revision: message.revision, - }); - } - }); - const expectCarryBeforeRoute = (revision: number): void => { - const events = parentAuthorityOrder.filter( - (event) => event.revision === revision, - ); - expect(events.length).toBeGreaterThan(0); - expect(events.length % 2).toBe(0); - for (let index = 0; index < events.length; index += 2) { - expect(events[index]).toEqual({ - type: "selected-edge-turn", - revision, - }); - expect(events[index + 1]).toEqual({ type: "route-update", revision }); - } - }; - const unrelatedViewer = await openClient(harness.webSocketUrl); - const unrelatedViewerAuth = peerAssisted( - await authenticate( - unrelatedViewer, - harness.room, - "viewer", - "selected-viewer-parent-unrelated-client", - 0, - ), - ); - expect(unrelatedViewerAuth.routeRevision).toBeGreaterThan( - childGrant.revision, - ); - const parentAfterUnrelatedJoin = await nextActiveRouteRevision( - prepared.rootViewer, - unrelatedViewerAuth.routeRevision, - ); - expect(parentAfterUnrelatedJoin.revision).toBeGreaterThan( - childGrant.revision, - ); - const [childCarry, parentCarry] = await Promise.all([ - prepared.failedViewer.inbox.next("selected-edge-turn"), - prepared.rootViewer.inbox.next("selected-edge-turn"), - ]); - expect(parentCarry).toEqual(childCarry); - expect(childCarry).toMatchObject({ - revision: parentAfterUnrelatedJoin.revision, - parentPeerId: childGrant.parentPeerId, - viewerPeerId: childGrant.viewerPeerId, - oldConnectionId: childGrant.oldConnectionId, - newConnectionId: childGrant.newConnectionId, - iceServer: childGrant.iceServer, - }); - expectCarryBeforeRoute(parentAfterUnrelatedJoin.revision); - parentAuthorityOrder.length = 0; - const replacementSibling = await openClient(harness.webSocketUrl); - const replacementSiblingAuth = peerAssisted( - await authenticate( - replacementSibling, - harness.room, - "viewer", - "selected-viewer-parent-root-sibling-client", - 0, - ), - ); - let sameRevisionCarry; - do { - sameRevisionCarry = await prepared.rootViewer.inbox.next( - "selected-edge-turn", - ); - } while (sameRevisionCarry.revision !== replacementSiblingAuth.routeRevision); - expect(sameRevisionCarry.newConnectionId).toBe(childGrant.newConnectionId); - await nextActiveRouteRevision( - prepared.rootViewer, - replacementSiblingAuth.routeRevision, - ); - expectCarryBeforeRoute(replacementSiblingAuth.routeRevision); - prepared.rootViewer.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: prepared.failedAuth.peerId, - payload: { - kind: "candidate", - connectionId: childGrant.newConnectionId, - candidate: { candidate: "candidate-after-sibling-reconnect" }, - }, - })); - expect(await prepared.failedViewer.inbox.next("signal")).toMatchObject({ - fromPeerId: prepared.rootAuth.peerId, - payload: { - kind: "candidate", - connectionId: childGrant.newConnectionId, - candidate: { candidate: "candidate-after-sibling-reconnect" }, - }, - }); - prepared.rootViewer.socket.send(JSON.stringify({ - type: "route-failed", - revision: replacementSiblingAuth.routeRevision, - phase: "active", - connectionId: childGrant.newConnectionId, - })); - await expect(prepared.rootViewer.inbox.next("error")).resolves.toMatchObject({ - code: "PEER_NOT_FOUND", - message: "Selected relay edge failed", - }); - prepared.rootViewer.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: prepared.failedAuth.peerId, - payload: { - kind: "candidate", - connectionId: childGrant.newConnectionId, - candidate: null, - }, - })); - expect((await prepared.rootViewer.inbox.next("error")).code).toBe("FORBIDDEN"); - }); - - it("moves both a failed deep viewer and its host branch within two SFU roots", async () => { - const issued: Array[0]> = []; - const harness = await startSfuHarness({ - tokenIssuer: { - async issueToken(request) { - issued.push(request); - return `token-${request.peerId}`; - }, - }, - }); - const peers = await exhaustDeepViewerPeerRoutes( - harness.webSocketUrl, - harness.room, - "deep-budget", - ); - - const hostPrepare = await nextPreparedRoute(peers.host); - const branchPrepare = await nextPreparedRoute(peers.secondRoot); - const failedPrepare = await nextPreparedRoute(peers.failedViewer); - expect(branchPrepare.revision).toBe(hostPrepare.revision); - expect(failedPrepare.revision).toBe(hostPrepare.revision); - expect(hostPrepare.assignment.childPeerIds).toEqual([ - peers.firstRootAuth.peerId, - ]); - expect(branchPrepare.assignment.upstream).toEqual({ kind: "sfu" }); - expect(failedPrepare.assignment.upstream).toEqual({ kind: "sfu" }); - expect( - new Set(issued.map(({ peerId }) => peerId)), - ).toEqual( - new Set([ - peers.hostAuth.peerId, - peers.secondRootAuth.peerId, - peers.failedAuth.peerId, - ]), - ); - }); - - it("fails a deep SFU fallback cleanly when deployment egress is exhausted", async () => { - let issuedTokens = 0; - const harness = await startSfuHarness({ - tokenIssuer: { - async issueToken(request) { - issuedTokens += 1; - return `token-${request.peerId}`; - }, - }, - sfuEgressCapacity: 1, - }); - const peers = await exhaustDeepViewerPeerRoutes( - harness.webSocketUrl, - harness.room, - "deep-limited", - ); - - expect((await peers.failedViewer.inbox.next("error")).code).toBe( - "PEER_NOT_FOUND", - ); - await expect(nextPreparedRoute(peers.host)).rejects.toThrow("Timed out"); - expect(issuedTokens).toBe(0); - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 0, egress: 0 }); - expect(peers.host.socket.readyState).toBe(WebSocket.OPEN); - }); - - it("admits a leaf with no peer slot through a selective SFU route", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { - async issueToken(request) { - return `token-${request.peerId}`; - }, - }, - }); - const host = await openClient(harness.webSocketUrl); - await authenticate(host, harness.room, "host", "admission-host"); - const firstLeaf = await openClient(harness.webSocketUrl); - const firstAuth = peerAssisted( - await authenticate( - firstLeaf, - harness.room, - "viewer", - "admission-leaf-a", - 0, - ), - ); - const secondLeaf = await openClient(harness.webSocketUrl); - const secondAuth = peerAssisted( - await authenticate( - secondLeaf, - harness.room, - "viewer", - "admission-leaf-b", - 0, - ), - ); - const pendingLeaf = await openClient(harness.webSocketUrl); - const pendingAuth = peerAssisted( - await authenticate( - pendingLeaf, - harness.room, - "viewer", - "admission-leaf-c", - 0, - ), - ); - expect(pendingAuth.routeAssignment.upstream).toEqual({ kind: "none" }); - - const hostPrepare = await nextPreparedRoute(host); - const budgetPrepare = await nextPreparedRoute(firstLeaf); - const pendingPrepare = await nextPreparedRoute(pendingLeaf); - expect(hostPrepare.assignment.childPeerIds).toEqual([secondAuth.peerId]); - expect(budgetPrepare.assignment.upstream).toEqual({ kind: "sfu" }); - expect(pendingPrepare.assignment.upstream).toEqual({ kind: "sfu" }); - expect(await firstLeaf.inbox.next("sfu-config")).toMatchObject({ - token: `token-${firstAuth.peerId}`, - }); - expect(await pendingLeaf.inbox.next("sfu-config")).toMatchObject({ - token: `token-${pendingAuth.peerId}`, - }); - }); - - it("uses a free direct slot beside an active SFU publication", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { - async issueToken(request) { - return `token-${request.peerId}`; - }, - }, - selectedEdgeTurn: true, - }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted( - await authenticate(host, harness.room, "host", "active-slot-host"), - ); - const root = await openClient(harness.webSocketUrl); - const rootAuth = peerAssisted( - await authenticate(root, harness.room, "viewer", "active-slot-root", 0), - ); - host.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: rootAuth.peerId, - payload: { - kind: "description", - connectionId: "active-slot-failed-edge", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await root.inbox.next("signal"); - root.socket.send( - JSON.stringify({ - type: "route-failed", - revision: rootAuth.routeRevision, - phase: "active", - connectionId: "active-slot-failed-edge", - }), - ); - const hostPrepare = await nextPreparedRoute(host); - const rootPrepare = await nextPreparedRoute(root); - await host.inbox.next("sfu-config"); - await root.inbox.next("sfu-config"); - host.socket.send( - JSON.stringify({ - type: "route-ready", - revision: hostPrepare.revision, - phase: "prepare", - }), - ); - root.socket.send( - JSON.stringify({ - type: "route-ready", - revision: rootPrepare.revision, - phase: "prepare", - }), - ); - await nextActiveRouteAfter(host, hostPrepare.revision - 1); - - const directViewer = await openClient(harness.webSocketUrl); - const directAuth = peerAssisted( - await authenticate( - directViewer, - harness.room, - "viewer", - "active-slot-direct", - 0, - ), - ); - expect(directAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: hostAuth.peerId, - }); - expect(directAuth.routeAssignment.sfuPublicationGeneration).toBeNull(); - root.socket.send(JSON.stringify({ - type: "refresh-sfu", - revision: directAuth.routeRevision, - })); - await root.inbox.next("sfu-config"); - root.socket.send(JSON.stringify({ - type: "route-failed", - revision: directAuth.routeRevision, - phase: "active", - connectionId: null, - })); - await expect(root.inbox.next("selected-edge-turn", 40)).rejects.toThrow( - "Timed out", - ); - }); - - it("retires an SFU publication when its root allowlist shrinks", async () => { - const issued: Array[0]> = []; - const harness = await startSfuHarness({ - tokenIssuer: { - async issueToken(request) { - issued.push(request); - return `token-${request.peerId}`; - }, - }, - }); - const peers = await exhaustDeepViewerPeerRoutes( - harness.webSocketUrl, - harness.room, - "shrink-roots", - ); - const hostPrepare = await nextPreparedRoute(peers.host); - const branchPrepare = await nextPreparedRoute(peers.secondRoot); - const failedPrepare = await nextPreparedRoute(peers.failedViewer); - await peers.host.inbox.next("sfu-config"); - await peers.secondRoot.inbox.next("sfu-config"); - await peers.failedViewer.inbox.next("sfu-config"); - for (const [client, revision] of [ - [peers.host, hostPrepare.revision], - [peers.secondRoot, branchPrepare.revision], - [peers.failedViewer, failedPrepare.revision], - ] as const) { - client.socket.send( - JSON.stringify({ type: "route-ready", revision, phase: "prepare" }), - ); - } - const hostActive = await nextActiveRouteAfter( - peers.host, - hostPrepare.revision - 1, - ); - await nextActiveRouteAfter( - peers.secondRoot, - branchPrepare.revision - 1, - ); - expect(hostActive.assignment.sfuPublicationGeneration).toBeTruthy(); - expect(issued).toHaveLength(3); - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 1, egress: 2 }); - - await closeClient(peers.failedViewer); - const retiredHost = await nextActiveRouteAfter( - peers.host, - hostActive.revision, - ); - const retiredRoot = await nextActiveRouteAfter( - peers.secondRoot, - hostActive.revision, - ); - expect(retiredHost.assignment.sfuPublicationGeneration).toBeNull(); - expect(retiredRoot.assignment).toMatchObject({ - upstream: { kind: "peer", peerId: peers.hostAuth.peerId }, - sfuPublicationGeneration: null, - }); - await new Promise((resolve) => setTimeout(resolve, 20)); - expect(issued).toHaveLength(3); - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 0, egress: 0 }); - }); - - it("separates host signaling reconnects from new sharing generations", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { - async issueToken(request) { - return `token-${request.peerId}`; - }, - }, - }); - const prepared = await prepareFallbackForTwoViewers( - harness.webSocketUrl, - harness.room, - "share-generation", - "share-gen-a", - ); - prepared.host.socket.send( - JSON.stringify({ - type: "route-ready", - revision: prepared.hostPrepare.revision, - phase: "prepare", - }), - ); - prepared.failedViewer.socket.send( - JSON.stringify({ - type: "route-ready", - revision: prepared.rootPrepare.revision, - phase: "prepare", - }), - ); - const active = await nextActiveRouteRevision( - prepared.failedViewer, - prepared.rootPrepare.revision, - ); - expect(active.assignment.upstream).toEqual({ kind: "sfu" }); - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 1, egress: 1 }); - - const sameShareHost = await openClient(harness.webSocketUrl); - const sameShareAuth = peerAssisted( - await authenticate( - sameShareHost, - harness.room, - "host", - "share-generation-host-client", - 1, - "share-gen-a", - ).catch((error: unknown) => { - throw new Error("same-generation authentication failed", { - cause: error, - }); - }), - ); - expect(sameShareAuth.routeRevision).toBe(active.revision); - expect(sameShareAuth.routeAssignment.sfuPublicationGeneration).toBeTruthy(); - await expect( - prepared.failedViewer.inbox.next("sharing-stopped", 30), - ).rejects.toThrow("Timed out"); - - const nextShareHost = await openClient(harness.webSocketUrl); - const nextShareAuth = peerAssisted( - await authenticate( - nextShareHost, - harness.room, - "host", - "share-generation-host-client", - 1, - "share-gen-b", - ).catch((error: unknown) => { - throw new Error("new-generation authentication failed", { - cause: error, - }); - }), - ); - await prepared.failedViewer.inbox.next("sharing-stopped"); - expect(nextShareAuth.routeRevision).toBeLessThanOrEqual(active.revision); - expect(nextShareAuth.routeAssignment.sfuPublicationGeneration).toBeNull(); - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 0, egress: 0 }); - - const staleStopClosed = new Promise((resolve) => - nextShareHost.socket.once("close", (code) => resolve(code)), - ); - nextShareHost.socket.send( - JSON.stringify({ - type: "stop-sharing", - shareGeneration: "share-gen-a", - }), - ); - expect(await staleStopClosed).toBe(4001); - await expect( - prepared.failedViewer.inbox.next("sharing-stopped", 30), - ).rejects.toThrow("Timed out"); - - const resumedSameShare = await openClient(harness.webSocketUrl); - const resumedAuth = peerAssisted( - await authenticate( - resumedSameShare, - harness.room, - "host", - "share-generation-host-client", - 1, - "share-gen-b", - ).catch((error: unknown) => { - throw new Error("resumed-generation authentication failed", { - cause: error, - }); - }), - ); - expect(resumedAuth.routeRevision).toBe(nextShareAuth.routeRevision); - }); - - it("replans a pending fallback after unrelated topology churn", async () => { - const generations: string[] = []; - const harness = await startSfuHarness({ - tokenIssuer: { - async issueToken(request) { - generations.push(request.publicationGeneration); - return `token-${request.peerId}`; - }, - }, - }); - const prepared = await prepareFallbackForTwoViewers( - harness.webSocketUrl, - harness.room, - "topology-churn", - ); - const extraViewer = await openClient(harness.webSocketUrl); - await authenticate( - extraViewer, - harness.room, - "viewer", - "topology-churn-extra", - 0, - ); - - let rollbackRevision: number | undefined; - let retriedPrepareRevision: number | undefined; - while ( - rollbackRevision === undefined || - retriedPrepareRevision === undefined - ) { - const message = await prepared.host.inbox.next("route-update"); - if ( - message.phase === "active" && - message.revision > prepared.hostPrepare.revision - ) { - rollbackRevision = message.revision; - } - if (message.phase === "prepare") { - retriedPrepareRevision = message.revision; - } - } - expect(retriedPrepareRevision).toBeGreaterThan(rollbackRevision); - expect(new Set(generations).size).toBe(2); - }); - - it("keeps a prepared fallback when an unrelated viewer disconnects", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { - async issueToken(request) { - return `token-${request.peerId}`; - }, - }, - viewerDisconnectGraceMs: 500, - }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted( - await authenticate(host, harness.room, "host", "unrelated-close-host"), - ); - const parentA = await openClient(harness.webSocketUrl); - const parentAAuth = peerAssisted( - await authenticate(parentA, harness.room, "viewer", "unrelated-parent-a"), - ); - const parentB = await openClient(harness.webSocketUrl); - await authenticate(parentB, harness.room, "viewer", "unrelated-parent-b"); - const failedViewer = await openClient(harness.webSocketUrl); - const failedAuth = peerAssisted( - await authenticate( - failedViewer, - harness.room, - "viewer", - "unrelated-failed", - ), - ); - const unrelatedViewer = await openClient(harness.webSocketUrl); - const unrelatedAuth = peerAssisted( - await authenticate( - unrelatedViewer, - harness.room, - "viewer", - "unrelated-disconnected", - 0, - ), - ); - expect(failedAuth.routeAssignment.upstream).toEqual({ - kind: "peer", - peerId: parentAAuth.peerId, - }); - - parentA.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: failedAuth.peerId, - payload: { - kind: "description", - connectionId: "unrelated-close-failed-edge", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await failedViewer.inbox.next("signal"); - failedViewer.socket.send( - JSON.stringify({ - type: "route-failed", - revision: unrelatedAuth.routeRevision, - phase: "active", - connectionId: "unrelated-close-failed-edge", - }), - ); - const hostPrepare = await nextPreparedRoute(host).catch((error: unknown) => { - throw new Error("host did not receive unrelated-close prepare", { - cause: error, - }); - }); - const failedPrepare = await nextPreparedRoute(failedViewer).catch( - (error: unknown) => { - throw new Error("failed viewer did not receive unrelated-close prepare", { - cause: error, - }); - }, - ); - const parentPrepare = await nextPreparedRoute(parentA); - await host.inbox.next("sfu-config"); - await failedViewer.inbox.next("sfu-config"); - await parentA.inbox.next("sfu-config"); - - await closeClient(unrelatedViewer); - host.socket.send( - JSON.stringify({ - type: "route-ready", - revision: hostPrepare.revision, - phase: "prepare", - }), - ); - failedViewer.socket.send( - JSON.stringify({ - type: "route-ready", - revision: failedPrepare.revision, - phase: "prepare", - }), - ); - parentA.socket.send( - JSON.stringify({ - type: "route-ready", - revision: parentPrepare.revision, - phase: "prepare", - }), - ); - const committed = await nextActiveRouteRevision( - host, - hostPrepare.revision, - ).catch((error: unknown) => { - throw new Error("unrelated close aborted the prepared route", { - cause: error, - }); - }); - expect(committed.assignment).toMatchObject({ - upstream: { kind: "none" }, - sfuPublicationGeneration: expect.any(String), - }); - expect(committed.assignment.childPeerIds.length).toBeLessThanOrEqual(1); - expect(hostAuth.peerId).toBeTruthy(); - }); - - it("rolls back SFU prepare on timeout and expected-session replacement", async () => { - const harness = await startSfuHarness({ - tokenIssuer: { - async issueToken(request) { - return `token-${request.peerId}`; - }, - }, - prepareTimeoutMs: 120, - }); - const timedOut = await prepareFallbackForTwoViewers( - harness.webSocketUrl, - harness.room, - "timeout", - ); - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 1, egress: 1 }); - const timedOutRollback = await nextActiveRouteAfter( - timedOut.host, - timedOut.hostPrepare.revision, - ); - expect(timedOutRollback).toMatchObject({ - revision: timedOut.hostPrepare.revision + 1, - assignment: { - upstream: { kind: "none" }, - childPeerIds: [timedOut.rootAuth.peerId], - sfuPublicationGeneration: null, - }, - }); - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 0, egress: 0 }); - - const replacementRoom = harness.roomStore.createRoom(); - const replacing = await prepareFallbackForTwoViewers( - harness.webSocketUrl, - replacementRoom, - "replacement", - ); - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 1, egress: 1 }); - const replacementRoot = await openClient(harness.webSocketUrl); - const replacementAuth = peerAssisted( - await authenticate( - replacementRoot, - replacementRoom, - "viewer", - "replacement-failed-client", - ), - ); - expect(replacementAuth).toMatchObject({ - peerId: replacing.failedAuth.peerId, - routeRevision: replacing.hostPrepare.revision + 1, - routeAssignment: { - upstream: { kind: "peer", peerId: replacing.rootAuth.peerId }, - sfuPublicationGeneration: null, - }, - }); - const replacementRollback = await nextActiveRouteAfter( - replacing.host, - replacing.hostPrepare.revision, - ); - expect(replacementRollback.revision).toBe( - replacing.hostPrepare.revision + 1, - ); - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 0, egress: 0 }); - - replacing.rootViewer.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: replacementAuth.peerId, - payload: { - kind: "description", - connectionId: "replacement-reported-again", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await replacementRoot.inbox.next("signal"); - replacementRoot.socket.send( - JSON.stringify({ - type: "route-failed", - revision: replacementAuth.routeRevision, - phase: "active", - connectionId: "replacement-reported-again", - }), - ); - const replacementRecovered = await nextActiveRouteAfter( - replacementRoot, - replacementAuth.routeRevision, - ); - expect(replacementRecovered.assignment.upstream).toEqual({ - kind: "peer", - peerId: replacing.hostAuth.peerId, - }); - - replacing.failedViewer.socket.send( - JSON.stringify({ - type: "route-ready", - revision: replacing.hostPrepare.revision, - phase: "prepare", - }), - ); - await new Promise((resolve) => setTimeout(resolve, 20)); - expect(replacementRoot.socket.readyState).toBe(WebSocket.OPEN); - - const prepareFailureRoom = harness.roomStore.createRoom(); - const prepareFailure = await prepareFallbackForTwoViewers( - harness.webSocketUrl, - prepareFailureRoom, - "prepare-failure", - ); - prepareFailure.failedViewer.socket.send( - JSON.stringify({ - type: "route-failed", - revision: prepareFailure.rootPrepare.revision, - phase: "prepare", - connectionId: null, - }), - ); - const failedPrepareRollback = await nextActiveRouteAfter( - prepareFailure.host, - prepareFailure.hostPrepare.revision, - ); - expect(failedPrepareRollback.revision).toBe( - prepareFailure.hostPrepare.revision + 1, - ); - expect(failedPrepareRollback.assignment).toMatchObject({ - childPeerIds: [prepareFailure.rootAuth.peerId], - sfuPublicationGeneration: null, - }); - }); - - it("fails back to peer routes when an active SFU token refresh fails", async () => { - let rejectTokens = false; - const harness = await startSfuHarness({ - tokenIssuer: { - async issueToken(request) { - if (rejectTokens) { - throw new Error("issuer unavailable"); - } - return `token-${request.peerId}`; - }, - }, - }); - const prepared = await prepareFallbackForTwoViewers( - harness.webSocketUrl, - harness.room, - "refresh-failure", - ); - prepared.host.socket.send( - JSON.stringify({ - type: "route-ready", - revision: prepared.hostPrepare.revision, - phase: "prepare", - }), - ); - prepared.failedViewer.socket.send( - JSON.stringify({ - type: "route-ready", - revision: prepared.rootPrepare.revision, - phase: "prepare", - }), - ); - await nextActiveRouteAfter( - prepared.host, - prepared.hostPrepare.revision - 1, - ); - await nextActiveRouteAfter( - prepared.failedViewer, - prepared.rootPrepare.revision - 1, - ); - - rejectTokens = true; - prepared.failedViewer.socket.send( - JSON.stringify({ - type: "refresh-sfu", - revision: prepared.rootPrepare.revision, - }), - ); - const rollback = await nextActiveRouteAfter( - prepared.host, - prepared.hostPrepare.revision, - ); - expect(rollback).toMatchObject({ - revision: prepared.hostPrepare.revision + 1, - assignment: { - childPeerIds: [prepared.rootAuth.peerId], - sfuPublicationGeneration: null, - }, - }); - }); - - it("keeps route revisions monotonic when a disconnected host is replaced", async () => { - const harness = await startHarness({ peerAssistedMedia: true }); - const firstHost = await openClient(harness.webSocketUrl); - const firstHostAuth = peerAssisted( - await authenticate(firstHost, harness.room, "host", "takeover-host-a"), - ); - const viewer = await openClient(harness.webSocketUrl); - const viewerAuth = peerAssisted( - await authenticate(viewer, harness.room, "viewer", "takeover-viewer-a"), - ); - const secondViewer = await openClient(harness.webSocketUrl); - await authenticate( - secondViewer, - harness.room, - "viewer", - "takeover-viewer-b", - ); - const previousRoute = await nextActiveRouteAfter( - viewer, - viewerAuth.routeRevision, - ); - - await closeClient(firstHost); - await viewer.inbox.next("host-status"); - const replacementHost = await openClient(harness.webSocketUrl); - const replacementAuth = peerAssisted( - await authenticate( - replacementHost, - harness.room, - "host", - "takeover-host-b", - ), - ); - expect(replacementAuth.peerId).not.toBe(firstHostAuth.peerId); - expect(replacementAuth.routeRevision).toBeGreaterThan( - previousRoute.revision, - ); - const viewerRoute = await nextActiveRouteAfter( - viewer, - previousRoute.revision, - ); - expect(viewerRoute).toMatchObject({ - revision: replacementAuth.routeRevision, - assignment: { - upstream: { kind: "peer", peerId: replacementAuth.peerId }, - }, - }); - }); - - it("keeps relay assignments through grace and reattaches only the direct child subtree", async () => { - const harness = await startHarness({ - peerAssistedMedia: true, - viewerDisconnectGraceMs: 200, - }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted( - await authenticate(host, harness.room, "host", "relay-grace-host"), - ); - - const firstViewer = await openClient(harness.webSocketUrl); - const firstAuth = peerAssisted( - await authenticate( - firstViewer, - harness.room, - "viewer", - "relay-grace-a", - ), - ); - await host.inbox.next("media-assignment"); - - const secondViewer = await openClient(harness.webSocketUrl); - const secondAuth = peerAssisted( - await authenticate( - secondViewer, - harness.room, - "viewer", - "relay-grace-b", - ), - ); - await host.inbox.next("media-assignment"); - - const thirdViewer = await openClient(harness.webSocketUrl); - const thirdAuth = peerAssisted( - await authenticate( - thirdViewer, - harness.room, - "viewer", - "relay-grace-c", - ), - ); - await firstViewer.inbox.next("media-assignment"); - - const fourthViewer = await openClient(harness.webSocketUrl); - await authenticate( - fourthViewer, - harness.room, - "viewer", - "relay-grace-d", - ); - await secondViewer.inbox.next("media-assignment"); - - const fifthViewer = await openClient(harness.webSocketUrl); - const fifthAuth = peerAssisted( - await authenticate( - fifthViewer, - harness.room, - "viewer", - "relay-grace-e", - ), - ); - const thirdWithChild = await thirdViewer.inbox.next("media-assignment"); - expect(thirdWithChild.mediaAssignment).toEqual({ - parentPeerId: firstAuth.peerId, - childPeerIds: [fifthAuth.peerId], - }); - - host.inbox.ignore("route-update"); - thirdViewer.inbox.ignore("route-update"); - await closeClient(firstViewer); - await host.inbox.expectNone(30); - await thirdViewer.inbox.expectNone(30); - - host.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: firstAuth.peerId, - payload: { - kind: "description", - connectionId: "offer-while-child-offline", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await host.inbox.expectNone(30); - - const reconnectedFirstViewer = await openClient(harness.webSocketUrl); - const reconnectedFirstAuth = peerAssisted( - await authenticate( - reconnectedFirstViewer, - harness.room, - "viewer", - "relay-grace-a", - ), - ); - expect(reconnectedFirstAuth).toMatchObject({ - peerId: firstAuth.peerId, - connectionId: null, - mediaAssignment: { - parentPeerId: hostAuth.peerId, - childPeerIds: [thirdAuth.peerId], - }, - }); - expect( - (await host.inbox.next("media-assignment", 500)).mediaAssignment, - ).toEqual({ - parentPeerId: null, - childPeerIds: [firstAuth.peerId, secondAuth.peerId], - }); - await closeClient(reconnectedFirstViewer); - - expect( - (await host.inbox.next("media-assignment", 500)).mediaAssignment, - ).toEqual({ - parentPeerId: null, - childPeerIds: [secondAuth.peerId, thirdAuth.peerId], - }); - expect( - (await thirdViewer.inbox.next("media-assignment", 500)).mediaAssignment, - ).toEqual({ - parentPeerId: hostAuth.peerId, - childPeerIds: [fifthAuth.peerId], - }); - fifthViewer.inbox.ignore("route-update"); - await fifthViewer.inbox.expectNone(30); - }); - - it("reattaches a subtree root that reconnects after its parent expires", async () => { - const harness = await startHarness({ - peerAssistedMedia: true, - viewerDisconnectGraceMs: 200, - }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted( - await authenticate(host, harness.room, "host", "relay-offline-host"), - ); - const firstViewer = await openClient(harness.webSocketUrl); - const firstAuth = peerAssisted( - await authenticate( - firstViewer, - harness.room, - "viewer", - "relay-offline-a", - ), - ); - await host.inbox.next("media-assignment"); - const secondViewer = await openClient(harness.webSocketUrl); - const secondAuth = peerAssisted( - await authenticate( - secondViewer, - harness.room, - "viewer", - "relay-offline-b", - ), - ); - await host.inbox.next("media-assignment"); - const subtreeRoot = await openClient(harness.webSocketUrl); - const subtreeRootAuth = peerAssisted( - await authenticate( - subtreeRoot, - harness.room, - "viewer", - "relay-offline-c", - ), - ); - expect(subtreeRootAuth.mediaAssignment.parentPeerId).toBe(firstAuth.peerId); - await firstViewer.inbox.next("media-assignment"); - - await closeClient(firstViewer); - await new Promise((resolve) => setTimeout(resolve, 80)); - await closeClient(subtreeRoot); - - expect( - (await host.inbox.next("media-assignment", 500)).mediaAssignment, - ).toEqual({ - parentPeerId: null, - childPeerIds: [secondAuth.peerId], - }); - - const reconnectedSubtreeRoot = await openClient(harness.webSocketUrl); - const reconnectedAuth = peerAssisted( - await authenticate( - reconnectedSubtreeRoot, - harness.room, - "viewer", - "relay-offline-c", - ), - ); - expect(reconnectedAuth).toMatchObject({ - peerId: subtreeRootAuth.peerId, - connectionId: null, - mediaAssignment: { - parentPeerId: hostAuth.peerId, - childPeerIds: [], - }, - }); - expect( - (await host.inbox.next("media-assignment", 500)).mediaAssignment, - ).toEqual({ - parentPeerId: null, - childPeerIds: [secondAuth.peerId, subtreeRootAuth.peerId], - }); - }); - - it("retains peer relay assignments but clears connection generations when sharing stops", async () => { - const harness = await startHarness({ - peerAssistedMedia: true, - viewerDisconnectGraceMs: 500, - }); - const host = await openClient(harness.webSocketUrl); - const hostAuth = peerAssisted( - await authenticate(host, harness.room, "host", "relay-stop-host"), - ); - const viewer = await openClient(harness.webSocketUrl); - const viewerAuth = peerAssisted( - await authenticate( - viewer, - harness.room, - "viewer", - "relay-stop-viewer", - ), - ); - await host.inbox.next("media-assignment"); - - host.socket.send( - JSON.stringify({ - type: "set-quality-settings", - qualitySettings: lowQualitySettings, - }), - ); - expect(await viewer.inbox.next("quality-settings")).toEqual({ - type: "quality-settings", - qualitySettings: lowQualitySettings, - }); - - host.socket.send( - JSON.stringify({ - type: "signal", - targetPeerId: viewerAuth.peerId, - payload: { - kind: "description", - connectionId: "connection-before-relay-stop", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - }), - ); - await viewer.inbox.next("signal"); - - const stoppedHostClosed = new Promise((resolve) => - host.socket.once("close", (code) => resolve(code)), - ); - host.socket.send(JSON.stringify({ type: "stop-sharing" })); - await viewer.inbox.next("sharing-stopped"); - expect(await viewer.inbox.next("host-status")).toMatchObject({ - online: false, - }); - expect(await stoppedHostClosed).toBe(1000); - - await closeClient(viewer); - const reconnectedViewer = await openClient(harness.webSocketUrl); - const reconnectedViewerAuth = peerAssisted( - await authenticate( - reconnectedViewer, - harness.room, - "viewer", - "relay-stop-viewer", - ), - ); - expect(reconnectedViewerAuth).toMatchObject({ - peerId: viewerAuth.peerId, - connectionId: null, - mediaAssignment: { - parentPeerId: hostAuth.peerId, - childPeerIds: [], - }, - qualitySettings: lowQualitySettings, - }); - - const resumedHost = await openClient(harness.webSocketUrl); - const resumedHostAuth = peerAssisted( - await authenticate( - resumedHost, - harness.room, - "host", - "relay-stop-host", - ), - ); - expect(resumedHostAuth).toMatchObject({ - peerId: hostAuth.peerId, - mediaAssignment: { - parentPeerId: null, - childPeerIds: [viewerAuth.peerId], - }, - qualitySettings: lowQualitySettings, - }); - expect(await reconnectedViewer.inbox.next("host-status")).toMatchObject({ - online: true, - }); - }); - it("keeps a viewer peer stable when it reconnects inside the grace period", async () => { const harness = await startHarness({ viewerDisconnectGraceMs: 80 }); const host = await openClient(harness.webSocketUrl); @@ -10317,50 +1754,6 @@ describe("WebSocket signaling", () => { }); }); - it("reasserts an authoritative parent route when a peer-assisted viewer reconnects", async () => { - const harness = await startHarness({ - peerAssistedMedia: true, - viewerDisconnectGraceMs: 300, - }); - const host = await openClient(harness.webSocketUrl); - peerAssisted( - await authenticate(host, harness.room, "host", "relay-reauth-host"), - ); - const originalViewer = await openClient(harness.webSocketUrl); - const originalAuth = peerAssisted( - await authenticate( - originalViewer, - harness.room, - "viewer", - "relay-reauth-viewer", - ), - ); - await host.inbox.next("media-assignment"); - await host.inbox.next("route-update"); - await closeClient(originalViewer); - - const reconnectedViewer = await openClient(harness.webSocketUrl); - const reconnectedAuth = peerAssisted( - await authenticate( - reconnectedViewer, - harness.room, - "viewer", - "relay-reauth-viewer", - ), - ); - expect(reconnectedAuth.peerId).toBe(originalAuth.peerId); - expect(reconnectedAuth.connectionId).toBeNull(); - expect(await host.inbox.next("route-update")).toMatchObject({ - revision: reconnectedAuth.routeRevision, - phase: "active", - assignment: { - upstream: { kind: "none" }, - childPeerIds: [originalAuth.peerId], - sfuPublicationGeneration: null, - }, - }); - }); - it("replays viewers when a host reconnects", async () => { const harness = await startHarness({ viewerDisconnectGraceMs: 500 }); const viewer = await openClient(harness.webSocketUrl); @@ -10725,257 +2118,4 @@ describe("WebSocket signaling", () => { expect(await rejectedUpgradeStatus(harness.webSocketUrl)).toBe(503); }); - it("keeps a draining generation charged and blocks a second room until deletion proof", async () => { - let issuedTokens = 0; - const roomControl = new FakeSfuRoomControl(); - const harness = await startSfuHarness({ - roomControl, - sfuIngressCapacity: 1, - sfuEgressCapacity: 1, - tokenIssuer: { - async issueToken({ peerId }) { - issuedTokens += 1; - return `token-${peerId}`; - }, - }, - }); - const active = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "draining-capacity", - ); - const resourceFence = roomControl.created[0]!; - const staleTokenRoom = managedSfuRoomName(resourceFence); - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 1, egress: 1 }); - - const deletion = deferredVoid(); - roomControl.deleteBarrier = deletion.promise; - active.host.socket.send(JSON.stringify({ - type: "stop-sharing", - shareGeneration: "draining-capacity-share", - })); - await active.viewer.inbox.next("sharing-stopped"); - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 1, egress: 1 }); - expect(roomControl.canJoin(staleTokenRoom)).toBe(true); - - const secondRoom = harness.roomStore.createRoom(); - const second = await failSingleViewerDirect( - harness.webSocketUrl, - secondRoom, - "draining-second-room", - ); - expect((await second.viewer.inbox.next("error")).message).toBe( - "SFU server capacity is exhausted", - ); - expect(roomControl.created).toHaveLength(1); - expect(issuedTokens).toBe(2); - - deletion.resolve(); - await vi.waitFor(() => - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 0, egress: 0 }), - ); - expect(roomControl.canJoin(staleTokenRoom)).toBe(false); - }); - - it("keeps two active rooms isolated under one deployment-wide owner", async () => { - const roomControl = new FakeSfuRoomControl(); - const harness = await startSfuHarness({ - roomControl, - sfuIngressCapacity: 2, - sfuEgressCapacity: 2, - tokenIssuer: { - async issueToken({ peerId }) { - return `token-${peerId}`; - }, - }, - }); - const first = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "two-room-first", - ); - const secondRoom = harness.roomStore.createRoom(); - const second = await activateSingleViewerSfu( - harness.webSocketUrl, - secondRoom, - "two-room-second", - ); - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 2, egress: 2 }); - expect(new Set(roomControl.created.map(({ roomId }) => roomId))).toEqual( - new Set([harness.room.roomId, secondRoom.roomId]), - ); - - first.host.socket.send(JSON.stringify({ - type: "stop-sharing", - shareGeneration: "two-room-first-share", - })); - await first.viewer.inbox.next("sharing-stopped"); - await vi.waitFor(() => - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 1, egress: 1 }), - ); - expect( - roomControl.rooms.has(managedSfuRoomName(roomControl.created[1]!)), - ).toBe(true); - - second.viewer.socket.send(JSON.stringify({ - type: "refresh-sfu", - revision: second.revision, - })); - expect(await second.viewer.inbox.next("sfu-config")).toMatchObject({ - revision: second.revision, - }); - }); - - it("keeps a signaling-offline Host charged while LiveKit is live and reclaims it after departure", async () => { - const roomControl = new FakeSfuRoomControl(); - const error = vi.spyOn(console, "error").mockImplementation(() => undefined); - const harness = await startSfuHarness({ - roomControl, - hostOfflineCheckMs: 20, - tokenIssuer: { - async issueToken({ peerId }) { - return `token-${peerId}`; - }, - }, - }); - const active = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "host-offline", - ); - const resourceFence = roomControl.created[0]!; - roomControl.seedRoom(resourceFence, ["host"]); - roomControl.failHostCheck = true; - - await closeClient(active.host); - await new Promise((resolve) => setTimeout(resolve, 55)); - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 1, egress: 1 }); - expect(roomControl.rooms.has(managedSfuRoomName(resourceFence))).toBe(true); - expect(error).toHaveBeenCalledWith( - "SFU Host-offline check failed; capacity remains reserved", - ); - - roomControl.failHostCheck = false; - await new Promise((resolve) => setTimeout(resolve, 55)); - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 1, egress: 1 }); - roomControl.rooms.get(managedSfuRoomName(resourceFence))?.delete("host"); - await vi.waitFor( - () => - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 0, egress: 0 }), - { timeout: 300 }, - ); - expect(roomControl.rooms.has(managedSfuRoomName(resourceFence))).toBe(false); - error.mockRestore(); - }); - - it("keeps failed room deletion charged until a retry proves absence", async () => { - const roomControl = new FakeSfuRoomControl(); - const error = vi.spyOn(console, "error").mockImplementation(() => undefined); - const harness = await startSfuHarness({ - roomControl, - drainRetryMs: 20, - tokenIssuer: { - async issueToken({ peerId }) { - return `token-${peerId}`; - }, - }, - }); - const active = await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "delete-retry", - ); - const resourceFence = roomControl.created[0]!; - roomControl.failDelete = true; - - active.host.socket.send(JSON.stringify({ - type: "stop-sharing", - shareGeneration: "delete-retry-share", - })); - await active.viewer.inbox.next("sharing-stopped"); - await new Promise((resolve) => setTimeout(resolve, 55)); - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 1, egress: 1 }); - expect(roomControl.rooms.has(managedSfuRoomName(resourceFence))).toBe(true); - expect(error).toHaveBeenCalledWith( - "SFU room drain failed; capacity remains reserved", - ); - - roomControl.failDelete = false; - await vi.waitFor( - () => - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 0, egress: 0 }), - { timeout: 300 }, - ); - expect(roomControl.rooms.has(managedSfuRoomName(resourceFence))).toBe(false); - error.mockRestore(); - }); - - it("waits for managed LiveKit rooms to drain before graceful shutdown completes", async () => { - const roomControl = new FakeSfuRoomControl(); - const harness = await startSfuHarness({ - roomControl, - tokenIssuer: { - async issueToken({ peerId }) { - return `token-${peerId}`; - }, - }, - }); - await activateSingleViewerSfu( - harness.webSocketUrl, - harness.room, - "graceful-drain", - ); - const secondRoom = harness.roomStore.createRoom(); - const secondHost = await openClient(harness.webSocketUrl); - await authenticate( - secondHost, - secondRoom, - "host", - "graceful-second-host", - 1, - "graceful-second-share", - ); - const secondViewer = await openClient(harness.webSocketUrl); - const secondViewerAuth = peerAssisted(await authenticate( - secondViewer, - secondRoom, - "viewer", - "graceful-second-viewer", - )); - secondHost.socket.send(JSON.stringify({ - type: "signal", - targetPeerId: secondViewerAuth.peerId, - payload: { - kind: "description", - connectionId: "graceful-second-direct", - description: { type: "offer", sdp: "v=0\r\n" }, - }, - })); - await secondViewer.inbox.next("signal"); - const deletion = deferredVoid(); - roomControl.deleteBarrier = deletion.promise; - - let closed = false; - const closing = runningServer!.close().then(() => { - closed = true; - }); - await new Promise((resolve) => setTimeout(resolve, 20)); - expect(closed).toBe(false); - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 1, egress: 1 }); - secondViewer.socket.send(JSON.stringify({ - type: "route-failed", - revision: secondViewerAuth.routeRevision, - phase: "active", - connectionId: "graceful-second-direct", - })); - await new Promise((resolve) => setTimeout(resolve, 20)); - expect(roomControl.created).toHaveLength(1); - - deletion.resolve(); - await closing; - runningServer = undefined; - expect(closed).toBe(true); - expect(harness.sfuAdmission?.usage()).toEqual({ ingress: 0, egress: 0 }); - expect(roomControl.created).toHaveLength(1); - }); }); diff --git a/tests/sfu-resource-admission.test.ts b/tests/sfu-resource-admission.test.ts index cbc5c8e..f367ef4 100644 --- a/tests/sfu-resource-admission.test.ts +++ b/tests/sfu-resource-admission.test.ts @@ -86,6 +86,19 @@ describe("SfuResourceAdmission", () => { expect(admission.beginDrain(stale)).toBe(false); }); + it("does not commit a publication without its first exact subscription", () => { + const admission = new SfuResourceAdmission({ + ingressCapacity: 1, + egressCapacity: 1, + }); + const pending = publication("1", "publication_pending"); + expect(admission.reservePublication(pending)).toBe(true); + expect(admission.commitPublication(pending)).toBeNull(); + expect(admission.beginDrain(pending)).toBe(true); + expect(admission.completeDrain(pending)).toBe(true); + expect(admission.usage()).toEqual({ ingress: 0, egress: 0 }); + }); + it("commits a new publication and its subscriptions before draining the old generation", () => { const admission = new SfuResourceAdmission({ ingressCapacity: 2, @@ -218,21 +231,4 @@ describe("SfuResourceAdmission", () => { expect(admission.usage()).toEqual({ ingress: 0, egress: 0 }); }); - it("keeps the aggregate API available while the router migrates", () => { - const admission = new SfuResourceAdmission({ - ingressCapacity: 2, - egressCapacity: 3, - }); - const first = publication("1", "publication_first"); - const second = publication("1", "publication_second"); - expect(admission.reserve(first, 1)).toBe(true); - expect(admission.reserve(first, 1)).toBe(true); - expect(admission.reserve(first, 2)).toBe(false); - expect(admission.commit(first)).toEqual([]); - expect(admission.reserve(second, 2)).toBe(true); - expect(admission.commit(second)).toEqual([first]); - expect(admission.usage()).toEqual({ ingress: 2, egress: 3 }); - expect(admission.completeDrain(first)).toBe(true); - expect(admission.usage()).toEqual({ ingress: 1, egress: 2 }); - }); });