refactor(route): reconcile SFU room lifecycle

src/server/app.ts

@@ -11,11 +11,13 @@ import {
 import { SiteAccess } from "./access-session.js";
 import { loadConfig, type ServerConfig } from "./config.js";
 import { createIceConfig } from "./ice.js";
+import type { SfuFallbackOptions } from "./hybrid-media-router.js";
 import type { SfuTokenIssuer } from "./livekit-token.js";
+import type { SfuRoomControl } from "./sfu-room-control.js";
 import { SfuResourceAdmission } from "./sfu-resource-admission.js";
 import { RoomDatabase } from "./room-database.js";
 import { RoomStore, RoomStoreError } from "./room-store.js";
-import { SignalingServer } from "./signaling.js";
+import { SignalingServer, type SignalingOptions } from "./signaling.js";
 
 export interface CreateServerOptions {
   config?: ServerConfig;
@@ -31,6 +33,7 @@ export interface CreateServerOptions {
   maxUnauthenticatedSignalConnections?: number;
   siteAccessTtlSeconds?: number;
   sfuTokenIssuer?: SfuTokenIssuer;
+  sfuRoomControl?: SfuRoomControl;
 }
 
 export interface ScreenerServer {
@@ -48,22 +51,33 @@ export async function createScreenerServer(
   const config = options.config ?? loadConfig();
   const now = options.now ?? Date.now;
   const livekitFallback = config.livekitFallback;
-  const sfuFallback = livekitFallback
-    ? {
-        url: livekitFallback.url,
-        tokenIssuer:
-          options.sfuTokenIssuer ??
-          new (await import("./livekit-token.js")).LiveKitTokenIssuer({
-            apiKey: livekitFallback.apiKey,
-            apiSecret: livekitFallback.apiSecret,
-            maxViewersPerRoom: config.maxViewersPerRoom,
-          }),
-        admission: new SfuResourceAdmission({
-          ingressCapacity: livekitFallback.ingressCapacity,
-          egressCapacity: livekitFallback.egressCapacity,
+  let sfuFallback: SfuFallbackOptions | undefined;
+  let sfuRoomControl: SfuRoomControl | undefined;
+  if (livekitFallback) {
+    sfuRoomControl =
+      options.sfuRoomControl ??
+      new (await import("./sfu-room-control.js")).LiveKitSfuRoomControl({
+        apiUrl: livekitFallback.apiUrl,
+        apiKey: livekitFallback.apiKey,
+        apiSecret: livekitFallback.apiSecret,
+        maxViewersPerRoom: config.maxViewersPerRoom,
+      });
+    sfuFallback = {
+      url: livekitFallback.url,
+      tokenIssuer:
+        options.sfuTokenIssuer ??
+        new (await import("./livekit-token.js")).LiveKitTokenIssuer({
+          apiKey: livekitFallback.apiKey,
+          apiSecret: livekitFallback.apiSecret,
+          maxViewersPerRoom: config.maxViewersPerRoom,
         }),
-      }
-    : undefined;
+      admission: new SfuResourceAdmission({
+        ingressCapacity: livekitFallback.ingressCapacity,
+        egressCapacity: livekitFallback.egressCapacity,
+      }),
+      roomControl: sfuRoomControl,
+    };
+  }
   const roomStore =
     options.roomStore ??
     new RoomStore({
@@ -89,7 +103,14 @@ export async function createScreenerServer(
 
   let frontendHandler: FrontendHandler | undefined;
   let vite: ViteDevServer | undefined;
+  let acceptingTraffic = false;
   const httpServer = createServer((request, response) => {
+    if (!acceptingTraffic) {
+      response.setHeader("Cache-Control", "no-store");
+      response.setHeader("Retry-After", "1");
+      sendJson(response, 503, { error: "Service starting" });
+      return;
+    }
     void handleRequest(
       request,
       response,
@@ -111,7 +132,7 @@ export async function createScreenerServer(
     });
   });
 
-  const signaling = new SignalingServer({
+  const signalingOptions: SignalingOptions = {
     server: httpServer,
     roomStore,
     peerAssistedMedia: config.peerAssistedMedia,
@@ -133,7 +154,10 @@ export async function createScreenerServer(
       options.maxUnauthenticatedSignalConnections,
     passThroughUnknownUpgrades:
       config.nodeEnv === "development" && options.serveFrontend !== false,
-  });
+  };
+  let signaling: SignalingServer | undefined;
+  let startupOperation: Promise<number> | undefined;
+  let closing = false;
 
   if (options.serveFrontend !== false) {
     if (config.nodeEnv === "development") {
@@ -176,42 +200,109 @@ export async function createScreenerServer(
   return {
     httpServer,
     roomStore,
-    listen(port = config.port, host = config.listenHost) {
-      return new Promise<number>((resolve, reject) => {
-        const onError = (error: Error) => {
-          httpServer.off("listening", onListening);
-          reject(error);
-        };
-        const onListening = () => {
-          httpServer.off("error", onError);
-          const address = httpServer.address();
-          if (!address || typeof address === "string") {
-            reject(new Error("Server did not bind to a TCP port"));
-            return;
+    async listen(port = config.port, host = config.listenHost) {
+      if (startupOperation) {
+        throw new Error("Screener server startup was already requested");
+      }
+      if (closing) {
+        throw new Error("Screener server is closing");
+      }
+      startupOperation = (async () => {
+        const boundPort = await bindHttpServer(httpServer, port, host);
+        try {
+          await sfuRoomControl?.initialize();
+          if (!closing) {
+            signaling = new SignalingServer(signalingOptions);
+            acceptingTraffic = true;
           }
-          resolve(address.port);
-        };
-        httpServer.once("error", onError);
-        httpServer.once("listening", onListening);
-        httpServer.listen(port, host);
-      });
+          return boundPort;
+        } catch (error) {
+          try {
+            await closeHttpServer(httpServer);
+          } catch (closeError) {
+            throw new AggregateError(
+              [error, closeError],
+              "Screener startup reconciliation failed",
+            );
+          }
+          throw error;
+        }
+      })();
+      return await startupOperation;
     },
     async close() {
+      acceptingTraffic = false;
+      closing = true;
+      const errors: unknown[] = [];
+      try {
+        await startupOperation;
+      } catch {
+        // The listen caller owns the startup error; shutdown still closes resources.
+      }
+      try {
+        await signaling?.close();
+      } catch (error) {
+        errors.push(error);
+      }
       try {
-        await signaling.close();
         if (httpServer.listening) {
           await new Promise<void>((resolve, reject) => {
             httpServer.close((error) => (error ? reject(error) : resolve()));
           });
         }
+      } catch (error) {
+        errors.push(error);
+      }
+      try {
         await vite?.close();
-      } finally {
+      } catch (error) {
+        errors.push(error);
+      }
+      try {
         roomStore.close();
+      } catch (error) {
+        errors.push(error);
+      }
+      if (errors.length > 0) {
+        throw new AggregateError(errors, "Screener server shutdown failed");
       }
     },
   };
 }
 
+function bindHttpServer(
+  httpServer: ReturnType<typeof createServer>,
+  port: number,
+  host: string,
+): Promise<number> {
+  return new Promise<number>((resolve, reject) => {
+    const onError = (error: Error) => {
+      httpServer.off("listening", onListening);
+      reject(error);
+    };
+    const onListening = () => {
+      httpServer.off("error", onError);
+      const address = httpServer.address();
+      if (!address || typeof address === "string") {
+        reject(new Error("Server did not bind to a TCP port"));
+        return;
+      }
+      resolve(address.port);
+    };
+    httpServer.once("error", onError);
+    httpServer.once("listening", onListening);
+    httpServer.listen(port, host);
+  });
+}
+
+function closeHttpServer(
+  httpServer: ReturnType<typeof createServer>,
+): Promise<void> {
+  return new Promise<void>((resolve, reject) => {
+    httpServer.close((error) => (error ? reject(error) : resolve()));
+  });
+}
+
 async function handleRequest(
   request: IncomingMessage,
   response: ServerResponse,

src/server/config.ts

@@ -35,6 +35,7 @@ const REMOVED_ENVIRONMENT_VARIABLES = [
 
 export interface LiveKitFallbackConfig {
   url: string;
+  apiUrl: string;
   apiKey: string;
   apiSecret: string;
   ingressCapacity: number;
@@ -139,9 +140,10 @@ function parseLiveKitFallback(
   nodeEnv: RuntimeEnvironment,
 ): LiveKitFallbackConfig | undefined {
   const url = environment.LIVEKIT_URL?.trim() || undefined;
+  const apiUrl = environment.LIVEKIT_API_URL?.trim() || undefined;
   const apiKey = environment.LIVEKIT_API_KEY?.trim() || undefined;
   const apiSecret = environment.LIVEKIT_API_SECRET?.trim() || undefined;
-  const configuredValues = [url, apiKey, apiSecret].filter(Boolean).length;
+  const configuredValues = [url, apiUrl, apiKey, apiSecret].filter(Boolean).length;
   const ingressCapacity = environment.SFU_INGRESS_CAPACITY?.trim() || undefined;
   const egressCapacity = environment.SFU_EGRESS_CAPACITY?.trim() || undefined;
   const configuredCapacities = [ingressCapacity, egressCapacity].filter(
@@ -156,9 +158,9 @@ function parseLiveKitFallback(
     }
     return undefined;
   }
-  if (configuredValues !== 3) {
+  if (configuredValues !== 4) {
     throw new Error(
-      "LIVEKIT_URL, LIVEKIT_API_KEY, and LIVEKIT_API_SECRET must be configured together",
+      "LIVEKIT_URL, LIVEKIT_API_URL, LIVEKIT_API_KEY, and LIVEKIT_API_SECRET must be configured together",
     );
   }
   if (configuredCapacities !== 2) {
@@ -194,8 +196,37 @@ function parseLiveKitFallback(
     throw new Error("LIVEKIT_URL must use wss in production");
   }
 
+  let parsedApiUrl: URL;
+  try {
+    parsedApiUrl = new URL(apiUrl!);
+  } catch {
+    throw new Error("LIVEKIT_API_URL must be a valid http or https origin");
+  }
+  if (parsedApiUrl.protocol !== "http:" && parsedApiUrl.protocol !== "https:") {
+    throw new Error("LIVEKIT_API_URL must use http or https");
+  }
+  if (
+    parsedApiUrl.username ||
+    parsedApiUrl.password ||
+    parsedApiUrl.pathname !== "/" ||
+    parsedApiUrl.search ||
+    parsedApiUrl.hash
+  ) {
+    throw new Error(
+      "LIVEKIT_API_URL must be an origin without credentials, path, query, or fragment",
+    );
+  }
+  if (
+    nodeEnv === "production" &&
+    parsedApiUrl.protocol !== "https:" &&
+    !isLoopbackHostname(parsedApiUrl.hostname)
+  ) {
+    throw new Error("LIVEKIT_API_URL must use https or loopback in production");
+  }
+
   return {
     url: parsedUrl.origin,
+    apiUrl: parsedApiUrl.origin,
     apiKey: apiKey!,
     apiSecret: apiSecret!,
     ingressCapacity: parseRequiredPositiveInteger(
@@ -209,6 +240,15 @@ function parseLiveKitFallback(
   };
 }
 
+function isLoopbackHostname(hostname: string): boolean {
+  return (
+    hostname === "localhost" ||
+    hostname === "127.0.0.1" ||
+    hostname === "[::1]" ||
+    hostname === "::1"
+  );
+}
+
 function parseUrlList(value: string | undefined, name: string): string[] {
   if (!value?.trim()) {
     return [];

src/server/hybrid-media-router.ts

@@ -21,6 +21,10 @@ import type {
   SfuResourceAdmission,
   SfuResourceFence,
 } from "./sfu-resource-admission.js";
+import {
+  managedSfuRoomName,
+  type SfuRoomControl,
+} from "./sfu-room-control.js";
 import type { SelectedEdgeTurnConfig } from "./config.js";
 import {
   MediaRouteController,
@@ -37,6 +41,8 @@ import { issueSelectedEdgeTurnCredential } from "./selected-edge-turn.js";
 type ErrorCode = Extract<ServerMessage, { type: "error" }>["code"];
 
 const DEFAULT_SFU_PREPARE_TIMEOUT_MS = 20_000;
+const DEFAULT_SFU_DRAIN_RETRY_MS = 1_000;
+const DEFAULT_HOST_OFFLINE_CHECK_MS = 5_000;
 const SELECTED_EDGE_TURN_ATTEMPT_TIMEOUT_MS = 30_000;
 const VIEWER_QUALITY_BAD_WINDOWS_TO_REASSIGN = 3;
 const VIEWER_QUALITY_EVIDENCE_GAP_MS = 5_000;
@@ -62,6 +68,18 @@ interface PendingRoutePreparation {
   timer: NodeJS.Timeout;
 }
 
+interface SfuDrainTask {
+  fence: SfuResourceFence;
+  operation?: Promise<void>;
+  retryTimer?: NodeJS.Timeout;
+}
+
+interface HostOfflineCheck {
+  hostPeerId: string;
+  fence: SfuResourceFence;
+  timer: NodeJS.Timeout;
+}
+
 interface ViewerRouteIntent {
   failedParentPeerId: string | null;
   failedParentSessionId?: string;
@@ -190,7 +208,10 @@ export interface SfuFallbackOptions {
   url: string;
   tokenIssuer: SfuTokenIssuer;
   admission: SfuResourceAdmission;
+  roomControl: SfuRoomControl;
   prepareTimeoutMs?: number;
+  drainRetryMs?: number;
+  hostOfflineCheckMs?: number;
 }
 
 export interface HybridMediaRouterOptions {
@@ -283,6 +304,9 @@ export class HybridMediaRouter {
     string,
     DeferredHealthySfuReselection
   >();
+  private readonly sfuDrainTasks = new Map<string, SfuDrainTask>();
+  private readonly hostOfflineChecks = new Map<string, HostOfflineCheck>();
+  private closing = false;
   constructor(private readonly options: HybridMediaRouterOptions) {
     assertEndpointMediaCopyCapacity(options.endpointMediaCopyCapacity);
     this.peerRelayTopology = new PeerRelayTopology(
@@ -303,10 +327,22 @@ export class HybridMediaRouter {
     ) {
       throw new Error("SFU fallback timeout is invalid");
     }
+    for (const [value, name] of [
+      [fallback.drainRetryMs, "SFU drain retry"],
+      [fallback.hostOfflineCheckMs, "SFU Host-offline check"],
+    ] as const) {
+      if (
+        value !== undefined &&
+        (!Number.isSafeInteger(value) || value <= 0)
+      ) {
+        throw new Error(`${name} interval is invalid`);
+      }
+    }
     new URL(fallback.url);
   }
 
-  close(): void {
+  async close(): Promise<void> {
+    this.closing = true;
     for (const pending of this.pendingRoutePreparations.values()) {
       clearTimeout(pending.timer);
     }
@@ -325,6 +361,10 @@ export class HybridMediaRouter {
       }
     }
     this.deferredHealthySfuReselections.clear();
+    for (const check of this.hostOfflineChecks.values()) {
+      clearTimeout(check.timer);
+    }
+    this.hostOfflineChecks.clear();
     this.activeSelectedSfuIngresses.clear();
     this.mediaRouteControllers.clear();
     this.failedParentPeerIdsByViewer.clear();
@@ -335,7 +375,37 @@ export class HybridMediaRouter {
     this.viewerQualityEvidenceStates.clear();
     this.heldQualityParentExclusionsByViewer.clear();
     this.roomQualityMigrationCooldownUntilMs.clear();
-    this.options.sfuFallback?.admission.releaseAll();
+    const fallback = this.options.sfuFallback;
+    if (!fallback) {
+      return;
+    }
+    for (const fence of fallback.admission.beginDrainAll()) {
+      this.scheduleSfuDrain(fence);
+    }
+    const results = await Promise.allSettled(
+      [...this.sfuDrainTasks].map(async ([key, task]) => {
+        if (task.retryTimer) {
+          clearTimeout(task.retryTimer);
+          task.retryTimer = undefined;
+        }
+        if (task.operation) {
+          await task.operation;
+        }
+        if (this.sfuDrainTasks.get(key) === task) {
+          await this.runSfuDrain(key, task);
+        }
+      }),
+    );
+    const failures = results
+      .filter((result): result is PromiseRejectedResult => result.status === "rejected")
+      .map((result) => result.reason);
+    const usage = fallback.admission.usage();
+    if (failures.length > 0 || usage.ingress !== 0 || usage.egress !== 0) {
+      throw new AggregateError(
+        failures,
+        "LiveKit rooms were not fully drained during shutdown",
+      );
+    }
   }
 
   connectParticipant(input: AuthenticatedRouteParticipant): HybridAuthenticationState {
@@ -348,6 +418,9 @@ export class HybridMediaRouter {
     this.clearRoomViewerQualityEvidenceStates(input.roomId);
     this.clearHeldQualityParentExclusion(input.roomId, input.peerId);
     this.clearSfuRefreshesForPeer(input.roomId, input.peerId);
+    if (input.role === "host") {
+      this.cancelHostOfflineCheck(input.roomId);
+    }
     if (input.role === "viewer") {
       this.clearDeferredHealthySfuReselectionForParticipant(
         input.roomId,
@@ -1507,6 +1580,9 @@ export class HybridMediaRouter {
     this.clearSelectedEdgeTurnsForParticipant(roomId, peerId);
     this.clearViewerQualityStateForParticipant(roomId, peerId);
     this.clearDeferredHealthySfuReselectionForParticipant(roomId, peerId);
+    if (this.peerRelayTopology.getHostPeerId(roomId) === peerId) {
+      this.scheduleHostOfflineCheck(roomId, peerId);
+    }
     const pending = this.pendingRoutePreparations.get(roomId);
     if (!pending || this.pendingSessionsAreCurrent(roomId, pending)) {
       return;
@@ -2823,7 +2899,7 @@ export class HybridMediaRouter {
         sfuRootPeerIds: rootPeerIds,
       });
     } catch {
-      fallback.admission.release(sfuResourceFence);
+      this.beginSfuDrain(sfuResourceFence);
       this.sendError(
         sourceSessionId,
         "SERVER_ERROR",
@@ -2832,7 +2908,7 @@ export class HybridMediaRouter {
       return "unavailable";
     }
     if (revision === undefined) {
-      fallback.admission.release(sfuResourceFence);
+      this.beginSfuDrain(sfuResourceFence);
       return "unavailable";
     }
 
@@ -2863,7 +2939,7 @@ export class HybridMediaRouter {
       roomId,
       controller,
       revision,
-      generation,
+      sfuResourceFence,
       rootPeerIds,
       expectedSessionIds,
     );
@@ -2874,7 +2950,7 @@ export class HybridMediaRouter {
     roomId: string,
     controller: MediaRouteController,
     revision: number,
-    generation: string,
+    sfuResourceFence: SfuResourceFence,
     rootPeerIds: readonly string[],
     expectedSessionIds: ReadonlyMap<string, string>,
   ): Promise<void> {
@@ -2883,6 +2959,24 @@ export class HybridMediaRouter {
       return;
     }
     try {
+      await fallback.roomControl.createRoom(sfuResourceFence);
+      let pending = this.pendingRoutePreparations.get(roomId);
+      let pendingRoute = controller.getPendingRoute();
+      if (
+        this.mediaRouteControllers.get(roomId) !== controller ||
+        pending?.revision !== revision ||
+        pendingRoute?.route.revision !== revision
+      ) {
+        this.beginSfuDrain(sfuResourceFence);
+        return;
+      }
+      if (!this.pendingSessionsAreCurrent(roomId, pending)) {
+        this.abortPendingRoute(roomId);
+        return;
+      }
+      if (this.abortStalePendingQualityIntent(roomId, pending)) {
+        return;
+      }
       const grants = await Promise.all(
         [...expectedSessionIds].map(async ([peerId, sessionId]) => ({
           peerId,
@@ -2894,18 +2988,20 @@ export class HybridMediaRouter {
                 ? "host"
                 : "viewer",
             peerId,
-            publicationGeneration: generation,
+            shareGeneration: sfuResourceFence.shareGeneration,
+            publicationGeneration: sfuResourceFence.publicationGeneration,
             allowlistedRootPeerIds: rootPeerIds,
           }),
         })),
       );
-      const pending = this.pendingRoutePreparations.get(roomId);
-      const pendingRoute = controller.getPendingRoute();
+      pending = this.pendingRoutePreparations.get(roomId);
+      pendingRoute = controller.getPendingRoute();
       if (
         this.mediaRouteControllers.get(roomId) !== controller ||
         pending?.revision !== revision ||
         pendingRoute?.route.revision !== revision
       ) {
+        this.beginSfuDrain(sfuResourceFence);
         return;
       }
       if (!this.pendingSessionsAreCurrent(roomId, pending)) {
@@ -2943,6 +3039,8 @@ export class HybridMediaRouter {
         pending?.revision === revision
       ) {
         this.abortPendingRoute(roomId);
+      } else {
+        this.beginSfuDrain(sfuResourceFence);
       }
     }
   }
@@ -2988,11 +3086,15 @@ export class HybridMediaRouter {
     if (!controller.commit(revision)) {
       return;
     }
-    if (
-      pending.sfuResourceFence &&
-      !this.options.sfuFallback?.admission.commit(pending.sfuResourceFence)
-    ) {
-      throw new Error("Committed SFU route has no resource reservation");
+    let supersededSfuFences: readonly SfuResourceFence[] = [];
+    if (pending.sfuResourceFence) {
+      const result = this.options.sfuFallback?.admission.commit(
+        pending.sfuResourceFence,
+      );
+      if (!result) {
+        throw new Error("Committed SFU route has no resource reservation");
+      }
+      supersededSfuFences = result;
     }
     clearTimeout(pending.timer);
     this.pendingRoutePreparations.delete(roomId);
@@ -3020,6 +3122,9 @@ export class HybridMediaRouter {
       this.clearChangedRouteConnectionIds(roomId, before, active);
     }
     this.broadcastActiveRoute(roomId, active);
+    for (const fence of supersededSfuFences) {
+      this.scheduleSfuDrain(fence);
+    }
     this.drainViewerRouteIntents(roomId);
   }
 
@@ -3192,7 +3297,7 @@ export class HybridMediaRouter {
     clearTimeout(pending.timer);
     this.pendingRoutePreparations.delete(roomId);
     if (pending.sfuResourceFence) {
-      this.options.sfuFallback?.admission.release(pending.sfuResourceFence);
+      this.beginSfuDrain(pending.sfuResourceFence);
     }
     if (pending.healthyReselection) {
       this.options.deleteConnectionId(roomId, pending.healthyReselection.rootPeerId);
@@ -3325,12 +3430,14 @@ export class HybridMediaRouter {
     const fallback = this.options.sfuFallback;
     const controller = this.mediaRouteControllers.get(roomId);
     const active = controller?.getActiveRoute();
+    const shareGeneration = this.options.getShareGeneration(roomId);
     if (
       !fallback ||
       !controller ||
       !active ||
       active.revision !== revision ||
       !active.sfu.publicationGeneration ||
+      !shareGeneration ||
       (peerId !== this.peerRelayTopology.getHostPeerId(roomId) &&
         !active.sfu.rootPeerIds.includes(peerId))
     ) {
@@ -3383,6 +3490,7 @@ export class HybridMediaRouter {
             ? "host"
             : "viewer",
         peerId,
+        shareGeneration,
         publicationGeneration: active.sfu.publicationGeneration,
         allowlistedRootPeerIds: active.sfu.rootPeerIds,
       });
@@ -3392,7 +3500,8 @@ export class HybridMediaRouter {
         this.connectedPeer(roomId, peerId)?.sessionId !== sessionId ||
         current.revision !== revision ||
         current.sfu.publicationGeneration !==
-          active.sfu.publicationGeneration
+          active.sfu.publicationGeneration ||
+        this.options.getShareGeneration(roomId) !== shareGeneration
       ) {
         return;
       }
@@ -3410,7 +3519,8 @@ export class HybridMediaRouter {
         this.connectedPeer(roomId, peerId)?.sessionId === sessionId &&
         current.revision === revision &&
         current.sfu.publicationGeneration ===
-          active.sfu.publicationGeneration
+          active.sfu.publicationGeneration &&
+        this.options.getShareGeneration(roomId) === shareGeneration
       ) {
         if (
           active.sfu.rootPeerIds.includes(peerId) &&
@@ -3522,7 +3632,7 @@ export class HybridMediaRouter {
     excludedPeerId?: string,
   ): void {
     if (route.sfu.publicationGeneration === null) {
-      this.options.sfuFallback?.admission.releaseRoom(roomId);
+      this.beginSfuRoomDrain(roomId);
     }
     this.reconcileSelectedPeerEdgeLeases(roomId, route);
     for (const [key, attempt] of this.selectedSfuIngressAttempts) {
@@ -3796,7 +3906,168 @@ export class HybridMediaRouter {
     }
   }
 
+  private beginSfuDrain(fence: SfuResourceFence): boolean {
+    const fallback = this.options.sfuFallback;
+    if (!fallback || !fallback.admission.beginDrain(fence)) {
+      return false;
+    }
+    this.scheduleSfuDrain(fence);
+    return true;
+  }
+
+  private beginSfuRoomDrain(roomId: string): void {
+    const fallback = this.options.sfuFallback;
+    if (!fallback) {
+      return;
+    }
+    for (const fence of fallback.admission.beginDrainRoom(roomId)) {
+      this.scheduleSfuDrain(fence);
+    }
+  }
+
+  private scheduleSfuDrain(fence: SfuResourceFence): void {
+    const fallback = this.options.sfuFallback;
+    if (!fallback) {
+      return;
+    }
+    const key = managedSfuRoomName(fence);
+    let task = this.sfuDrainTasks.get(key);
+    if (!task) {
+      task = { fence: { ...fence } };
+      this.sfuDrainTasks.set(key, task);
+    }
+    if (!task.operation && !task.retryTimer) {
+      void this.runSfuDrain(key, task).catch(() => undefined);
+    }
+  }
+
+  private runSfuDrain(key: string, task: SfuDrainTask): Promise<void> {
+    if (task.operation) {
+      return task.operation;
+    }
+    const fallback = this.options.sfuFallback;
+    if (!fallback) {
+      return Promise.resolve();
+    }
+    let tracked!: Promise<void>;
+    tracked = (async () => {
+      try {
+        await fallback.roomControl.deleteRoom(task.fence);
+        if (this.sfuDrainTasks.get(key) !== task) {
+          return;
+        }
+        if (!fallback.admission.completeDrain(task.fence)) {
+          throw new Error("LiveKit drain has no matching resource generation");
+        }
+        if (task.retryTimer) {
+          clearTimeout(task.retryTimer);
+        }
+        this.sfuDrainTasks.delete(key);
+      } catch (error) {
+        if (this.sfuDrainTasks.get(key) !== task) {
+          return;
+        }
+        if (this.closing) {
+          throw error;
+        }
+        console.error("SFU room drain failed; capacity remains reserved");
+        if (!task.retryTimer) {
+          task.retryTimer = setTimeout(() => {
+            task.retryTimer = undefined;
+            void this.runSfuDrain(key, task).catch(() => undefined);
+          }, fallback.drainRetryMs ?? DEFAULT_SFU_DRAIN_RETRY_MS);
+          task.retryTimer.unref();
+        }
+      }
+    })().finally(() => {
+      if (task.operation === tracked) {
+        task.operation = undefined;
+      }
+    });
+    task.operation = tracked;
+    return tracked;
+  }
+
+  private activeSfuFence(roomId: string): SfuResourceFence | null {
+    const publicationGeneration = this.mediaRouteControllers
+      .get(roomId)
+      ?.getActiveRoute().sfu.publicationGeneration;
+    const shareGeneration = this.options.getShareGeneration(roomId);
+    return publicationGeneration && shareGeneration
+      ? { roomId, shareGeneration, publicationGeneration }
+      : null;
+  }
+
+  private scheduleHostOfflineCheck(roomId: string, hostPeerId: string): void {
+    this.cancelHostOfflineCheck(roomId);
+    const fallback = this.options.sfuFallback;
+    const fence = this.activeSfuFence(roomId);
+    if (!fallback || !fence || this.closing) {
+      return;
+    }
+    const timer = setTimeout(() => {
+      void this.checkHostOffline(roomId).catch(() => undefined);
+    }, fallback.hostOfflineCheckMs ?? DEFAULT_HOST_OFFLINE_CHECK_MS);
+    timer.unref();
+    this.hostOfflineChecks.set(roomId, {
+      hostPeerId,
+      fence,
+      timer,
+    });
+  }
+
+  private async checkHostOffline(roomId: string): Promise<void> {
+    const check = this.hostOfflineChecks.get(roomId);
+    const fallback = this.options.sfuFallback;
+    if (!check || !fallback || this.closing) {
+      return;
+    }
+    if (this.options.roomStore.getConnectedHost(roomId)) {
+      this.cancelHostOfflineCheck(roomId);
+      return;
+    }
+
+    let hostExists: boolean;
+    try {
+      hostExists = await fallback.roomControl.hostParticipantExists(check.fence);
+    } catch {
+      if (this.hostOfflineChecks.get(roomId) === check) {
+        console.error("SFU Host-offline check failed; capacity remains reserved");
+        this.hostOfflineChecks.delete(roomId);
+        this.scheduleHostOfflineCheck(roomId, check.hostPeerId);
+      }
+      return;
+    }
+    if (this.hostOfflineChecks.get(roomId) !== check) {
+      return;
+    }
+    if (this.options.roomStore.getConnectedHost(roomId)) {
+      this.cancelHostOfflineCheck(roomId);
+      return;
+    }
+    const currentFence = this.activeSfuFence(roomId);
+    if (!currentFence || !sameSfuFence(currentFence, check.fence)) {
+      this.cancelHostOfflineCheck(roomId);
+      return;
+    }
+    this.hostOfflineChecks.delete(roomId);
+    if (hostExists) {
+      this.scheduleHostOfflineCheck(roomId, check.hostPeerId);
+      return;
+    }
+    this.clearRoomMediaRouteState(roomId);
+  }
+
+  private cancelHostOfflineCheck(roomId: string): void {
+    const check = this.hostOfflineChecks.get(roomId);
+    if (check) {
+      clearTimeout(check.timer);
+      this.hostOfflineChecks.delete(roomId);
+    }
+  }
+
   private clearRoomMediaRouteState(roomId: string): void {
+    this.cancelHostOfflineCheck(roomId);
     const pending = this.pendingRoutePreparations.get(roomId);
     if (pending) {
       clearTimeout(pending.timer);
@@ -3821,7 +4092,7 @@ export class HybridMediaRouter {
     this.sfuDisabledRoomIds.delete(roomId);
     this.clearRoomViewerQualityEvidenceStates(roomId);
     this.roomQualityMigrationCooldownUntilMs.delete(roomId);
-    this.options.sfuFallback?.admission.releaseRoom(roomId);
+    this.beginSfuRoomDrain(roomId);
   }
 
   private clearSelectedEdgeTurnsForParticipant(
@@ -4049,6 +4320,14 @@ function viewerConnectionKey(roomId: string, peerId: string): string {
   return `${roomId}:${peerId}`;
 }
 
+function sameSfuFence(left: SfuResourceFence, right: SfuResourceFence): boolean {
+  return (
+    left.roomId === right.roomId &&
+    left.shareGeneration === right.shareGeneration &&
+    left.publicationGeneration === right.publicationGeneration
+  );
+}
+
 function sameQualityRouteIntentGuard(
   left: QualityRouteIntentGuard,
   right: QualityRouteIntentGuard,

src/server/livekit-token.ts

@@ -1,6 +1,5 @@
 import {
   AccessToken,
-  RoomConfiguration,
   TrackSource,
 } from "livekit-server-sdk";
 
@@ -8,6 +7,7 @@ import {
   MAX_VIEWERS_PER_ROOM_LIMIT,
   type Role,
 } from "../shared/protocol.js";
+import { managedSfuRoomName } from "./sfu-room-control.js";
 
 const LIVEKIT_TOKEN_TTL_SECONDS = 5 * 60;
 const ROOM_ID_PATTERN = /^[1-9]\d{0,11}$/;
@@ -17,6 +17,7 @@ export interface SfuTokenRequest {
   roomId: string;
   role: Role;
   peerId: string;
+  shareGeneration: string;
   publicationGeneration: string;
   allowlistedRootPeerIds: readonly string[];
 }
@@ -49,10 +50,7 @@ export class LiveKitTokenIssuer implements SfuTokenIssuer {
     validateTokenRequest(request, this.options.maxViewersPerRoom);
 
     const isHost = request.role === "host";
-    const room = liveKitRoomName(
-      request.roomId,
-      request.publicationGeneration,
-    );
+    const room = managedSfuRoomName(request);
     const token = new AccessToken(
       this.options.apiKey,
       this.options.apiSecret,
@@ -72,11 +70,6 @@ export class LiveKitTokenIssuer implements SfuTokenIssuer {
       canPublishData: false,
       canUpdateOwnMetadata: false,
     });
-    token.roomConfig = new RoomConfiguration({
-      name: room,
-      maxParticipants: this.options.maxViewersPerRoom + 1,
-    });
-
     return await token.toJwt();
   }
 }
@@ -90,6 +83,7 @@ function validateTokenRequest(
   }
   if (
     !OPAQUE_ID_PATTERN.test(request.peerId) ||
+    !OPAQUE_ID_PATTERN.test(request.shareGeneration) ||
     !OPAQUE_ID_PATTERN.test(request.publicationGeneration)
   ) {
     throw new Error("LiveKit participant identity is invalid");
@@ -111,10 +105,3 @@ function validateTokenRequest(
     throw new Error("LiveKit viewer is not an allowlisted SFU root");
   }
 }
-
-function liveKitRoomName(
-  roomId: string,
-  publicationGeneration: string,
-): string {
-  return `screener-${roomId}-${publicationGeneration}`;
-}

src/server/sfu-resource-admission.ts

@@ -22,6 +22,7 @@ interface SfuResourceEntry {
 interface RoomSfuResources {
   committed?: SfuResourceEntry;
   reserved?: SfuResourceEntry;
+  draining: Map<string, SfuResourceEntry>;
 }
 
 export class SfuResourceAdmission {
@@ -48,6 +49,9 @@ export class SfuResourceAdmission {
     if (room?.committed && sameFence(room.committed.fence, fence)) {
       return room.committed.egress === egress;
     }
+    if (room?.draining.has(fenceKey(fence))) {
+      return false;
+    }
     if (
       this.ingressInUse >= this.ingressCapacity ||
       egress > this.egressCapacity - this.egressInUse
@@ -55,7 +59,7 @@ export class SfuResourceAdmission {
       return false;
     }
 
-    const resources = room ?? {};
+    const resources: RoomSfuResources = room ?? { draining: new Map() };
     resources.reserved = { fence: { ...fence }, egress };
     this.rooms.set(fence.roomId, resources);
     this.ingressInUse += 1;
@@ -63,68 +67,88 @@ export class SfuResourceAdmission {
     return true;
   }
 
-  commit(fence: SfuResourceFence): boolean {
+  commit(fence: SfuResourceFence): readonly SfuResourceFence[] | null {
     assertFence(fence);
     const room = this.rooms.get(fence.roomId);
     if (room?.committed && sameFence(room.committed.fence, fence)) {
-      return true;
+      return [];
     }
     if (!room?.reserved || !sameFence(room.reserved.fence, fence)) {
-      return false;
+      return null;
     }
 
+    const draining: SfuResourceFence[] = [];
     if (room.committed) {
-      this.subtract(room.committed);
+      this.addDraining(room, room.committed);
+      draining.push({ ...room.committed.fence });
     }
     room.committed = room.reserved;
     room.reserved = undefined;
-    return true;
+    return draining;
   }
 
-  release(fence: SfuResourceFence): boolean {
+  beginDrain(fence: SfuResourceFence): boolean {
     assertFence(fence);
     const room = this.rooms.get(fence.roomId);
     if (!room) {
       return false;
     }
 
-    let released = false;
+    if (room.draining.has(fenceKey(fence))) {
+      return true;
+    }
+    let entry: SfuResourceEntry | undefined;
     if (room.reserved && sameFence(room.reserved.fence, fence)) {
-      this.subtract(room.reserved);
+      entry = room.reserved;
       room.reserved = undefined;
-      released = true;
     }
     if (room.committed && sameFence(room.committed.fence, fence)) {
-      this.subtract(room.committed);
+      entry = room.committed;
       room.committed = undefined;
-      released = true;
     }
-    this.deleteEmptyRoom(fence.roomId, room);
-    return released;
+    if (!entry) {
+      return false;
+    }
+    this.addDraining(room, entry);
+    return true;
   }
 
-  releaseRoom(roomId: string): boolean {
+  completeDrain(fence: SfuResourceFence): boolean {
+    assertFence(fence);
+    const room = this.rooms.get(fence.roomId);
+    const entry = room?.draining.get(fenceKey(fence));
+    if (!room || !entry) {
+      return false;
+    }
+    room.draining.delete(fenceKey(fence));
+    this.subtract(entry);
+    this.deleteEmptyRoom(fence.roomId, room);
+    return true;
+  }
+
+  beginDrainRoom(roomId: string): readonly SfuResourceFence[] {
     if (!roomId) {
       throw new Error("SFU resource room ID is invalid");
     }
     const room = this.rooms.get(roomId);
     if (!room) {
-      return false;
+      return [];
     }
     if (room.reserved) {
-      this.subtract(room.reserved);
+      this.addDraining(room, room.reserved);
+      room.reserved = undefined;
     }
     if (room.committed) {
-      this.subtract(room.committed);
+      this.addDraining(room, room.committed);
+      room.committed = undefined;
     }
-    this.rooms.delete(roomId);
-    return true;
+    return [...room.draining.values()].map((entry) => ({ ...entry.fence }));
   }
 
-  releaseAll(): void {
-    this.rooms.clear();
-    this.ingressInUse = 0;
-    this.egressInUse = 0;
+  beginDrainAll(): readonly SfuResourceFence[] {
+    return [...this.rooms.keys()].flatMap((roomId) =>
+      this.beginDrainRoom(roomId),
+    );
   }
 
   usage(): SfuResourceUsage {
@@ -139,13 +163,21 @@ export class SfuResourceAdmission {
     }
   }
 
+  private addDraining(room: RoomSfuResources, entry: SfuResourceEntry): void {
+    room.draining.set(fenceKey(entry.fence), entry);
+  }
+
   private deleteEmptyRoom(roomId: string, room: RoomSfuResources): void {
-    if (!room.committed && !room.reserved) {
+    if (!room.committed && !room.reserved && room.draining.size === 0) {
       this.rooms.delete(roomId);
     }
   }
 }
 
+function fenceKey(fence: SfuResourceFence): string {
+  return `${fence.shareGeneration}\u0000${fence.publicationGeneration}`;
+}
+
 function sameEntry(
   entry: SfuResourceEntry,
   fence: SfuResourceFence,

src/server/sfu-room-control.ts

@@ -0,0 +1,193 @@
+import { RoomServiceClient, ServerError } from "livekit-server-sdk";
+
+import { MAX_VIEWERS_PER_ROOM_LIMIT } from "../shared/protocol.js";
+import type { SfuResourceFence } from "./sfu-resource-admission.js";
+
+const MANAGED_ROOM_PREFIX = "screener-v1.";
+const ROOM_ID_PATTERN = /^[1-9]\d{0,11}$/;
+const OPAQUE_ID_PATTERN = /^[A-Za-z0-9_-]{8,128}$/;
+const LEGACY_MANAGED_ROOM_PATTERN = /^screener-[1-9]\d{0,11}-[A-Za-z0-9_-]{8,128}$/;
+const ROOM_SERVICE_TIMEOUT_SECONDS = 5;
+
+interface LiveKitRoomRecord {
+  name: string;
+}
+
+interface LiveKitParticipantRecord {
+  identity: string;
+}
+
+export interface LiveKitRoomService {
+  createRoom(options: {
+    name: string;
+    maxParticipants: number;
+  }): Promise<LiveKitRoomRecord>;
+  listRooms(names?: string[]): Promise<LiveKitRoomRecord[]>;
+  deleteRoom(room: string): Promise<void>;
+  listParticipants(room: string): Promise<LiveKitParticipantRecord[]>;
+}
+
+export interface SfuRoomControl {
+  initialize(): Promise<void>;
+  createRoom(fence: SfuResourceFence): Promise<void>;
+  deleteRoom(fence: SfuResourceFence): Promise<void>;
+  hostParticipantExists(fence: SfuResourceFence): Promise<boolean>;
+}
+
+export interface LiveKitSfuRoomControlOptions {
+  apiUrl: string;
+  apiKey: string;
+  apiSecret: string;
+  maxViewersPerRoom: number;
+  roomService?: LiveKitRoomService;
+}
+
+export class LiveKitSfuRoomControl implements SfuRoomControl {
+  private readonly roomService: LiveKitRoomService;
+  private readonly maxParticipants: number;
+  private readonly roomOperations = new Map<string, Promise<void>>();
+
+  constructor(options: LiveKitSfuRoomControlOptions) {
+    if (!options.apiUrl || !options.apiKey || Buffer.byteLength(options.apiSecret) < 32) {
+      throw new Error("LiveKit RoomService configuration is invalid");
+    }
+    if (
+      !Number.isSafeInteger(options.maxViewersPerRoom) ||
+      options.maxViewersPerRoom < 1 ||
+      options.maxViewersPerRoom > MAX_VIEWERS_PER_ROOM_LIMIT
+    ) {
+      throw new Error("LiveKit viewer limit is invalid");
+    }
+    this.maxParticipants = options.maxViewersPerRoom + 1;
+    this.roomService =
+      options.roomService ??
+      new RoomServiceClient(
+        options.apiUrl,
+        options.apiKey,
+        options.apiSecret,
+        {
+          requestTimeout: ROOM_SERVICE_TIMEOUT_SECONDS,
+          failover: false,
+        },
+      );
+  }
+
+  async initialize(): Promise<void> {
+    const rooms = await this.roomService.listRooms();
+    if (rooms.some((room) => !isManagedSfuRoomName(room.name))) {
+      throw new Error("Dedicated LiveKit instance contains a foreign room");
+    }
+    for (const room of rooms) {
+      await this.deleteRoomByName(room.name);
+    }
+    if ((await this.roomService.listRooms()).length !== 0) {
+      throw new Error("Dedicated LiveKit instance is not empty after startup drain");
+    }
+  }
+
+  createRoom(fence: SfuResourceFence): Promise<void> {
+    const roomName = managedSfuRoomName(fence);
+    return this.serialize(roomName, async () => {
+      if ((await this.roomService.listRooms([roomName])).length !== 0) {
+        throw new Error("Managed LiveKit room already exists");
+      }
+      await this.roomService.createRoom({
+        name: roomName,
+        maxParticipants: this.maxParticipants,
+      });
+      const rooms = await this.roomService.listRooms([roomName]);
+      if (rooms.length !== 1 || rooms[0]?.name !== roomName) {
+        throw new Error("Managed LiveKit room creation was not confirmed");
+      }
+    });
+  }
+
+  deleteRoom(fence: SfuResourceFence): Promise<void> {
+    const roomName = managedSfuRoomName(fence);
+    return this.serialize(roomName, async () => {
+      await this.deleteRoomByName(roomName);
+      if ((await this.roomService.listRooms([roomName])).length !== 0) {
+        throw new Error("Managed LiveKit room deletion was not confirmed");
+      }
+    });
+  }
+
+  hostParticipantExists(fence: SfuResourceFence): Promise<boolean> {
+    const roomName = managedSfuRoomName(fence);
+    return this.serialize(roomName, async () => {
+      try {
+        return (await this.roomService.listParticipants(roomName)).some(
+          (participant) => participant.identity === "host",
+        );
+      } catch (error) {
+        if (isNotFound(error)) {
+          return false;
+        }
+        throw error;
+      }
+    });
+  }
+
+  private async deleteRoomByName(roomName: string): Promise<void> {
+    try {
+      await this.roomService.deleteRoom(roomName);
+    } catch (error) {
+      if (!isNotFound(error)) {
+        throw error;
+      }
+    }
+  }
+
+  private serialize<Result>(
+    roomName: string,
+    operation: () => Promise<Result>,
+  ): Promise<Result> {
+    const previous = this.roomOperations.get(roomName) ?? Promise.resolve();
+    const result = previous.catch(() => undefined).then(operation);
+    const tail = result.then(
+      () => undefined,
+      () => undefined,
+    );
+    this.roomOperations.set(roomName, tail);
+    void tail.then(() => {
+      if (this.roomOperations.get(roomName) === tail) {
+        this.roomOperations.delete(roomName);
+      }
+    });
+    return result;
+  }
+}
+
+export function managedSfuRoomName(fence: SfuResourceFence): string {
+  if (
+    !ROOM_ID_PATTERN.test(fence.roomId) ||
+    !OPAQUE_ID_PATTERN.test(fence.shareGeneration) ||
+    !OPAQUE_ID_PATTERN.test(fence.publicationGeneration)
+  ) {
+    throw new Error("Managed LiveKit room fence is invalid");
+  }
+  return `${MANAGED_ROOM_PREFIX}${fence.roomId}.${fence.shareGeneration}.${fence.publicationGeneration}`;
+}
+
+export function isManagedSfuRoomName(roomName: string): boolean {
+  if (LEGACY_MANAGED_ROOM_PATTERN.test(roomName)) {
+    return true;
+  }
+  if (!roomName.startsWith(MANAGED_ROOM_PREFIX)) {
+    return false;
+  }
+  const parts = roomName.slice(MANAGED_ROOM_PREFIX.length).split(".");
+  return (
+    parts.length === 3 &&
+    ROOM_ID_PATTERN.test(parts[0] ?? "") &&
+    OPAQUE_ID_PATTERN.test(parts[1] ?? "") &&
+    OPAQUE_ID_PATTERN.test(parts[2] ?? "")
+  );
+}
+
+function isNotFound(error: unknown): boolean {
+  return (
+    error instanceof ServerError &&
+    (error.status === 404 || error.code === "not_found")
+  );
+}

src/server/signaling.ts

@@ -216,6 +216,10 @@ export class SignalingServer {
         }
         return;
       }
+      if (this.closing) {
+        rejectUpgrade(socket, 503, "Service Unavailable");
+        return;
+      }
       if (!this.isAllowedOrigin(request.headers.origin)) {
         rejectUpgrade(socket, 403, "Forbidden");
         return;
@@ -250,26 +254,33 @@ export class SignalingServer {
 
   async close(): Promise<void> {
     this.closing = true;
+    this.options.server.off("upgrade", this.upgradeHandler);
     clearInterval(this.heartbeatTimer);
     clearInterval(this.cleanupTimer);
     for (const timer of this.viewerGraceTimers.values()) {
       clearTimeout(timer);
     }
     this.viewerGraceTimers.clear();
-    this.hybridMediaRouter?.close();
+    let routeCloseError: unknown;
+    try {
+      await this.hybridMediaRouter?.close();
+    } catch (error) {
+      routeCloseError = error;
+    }
     this.viewerQualityEvidenceGates.clear();
     this.viewerMediaReadyByRoom.clear();
     this.shareGenerationsByRoom.clear();
     this.pausedShareGenerationsByRoom.clear();
     this.ordinaryActiveHostChildrenByRoom.clear();
-    this.options.server.off("upgrade", this.upgradeHandler);
-
     for (const socket of this.webSocketServer.clients) {
       socket.terminate();
     }
     await new Promise<void>((resolve) => {
       this.webSocketServer.close(() => resolve());
     });
+    if (routeCloseError) {
+      throw routeCloseError;
+    }
   }
 
   private accept(socket: WebSocket, siteAccessAuthenticated: boolean): void {
@@ -313,7 +324,7 @@ export class SignalingServer {
 
   private handleMessage(socket: WebSocket, encoded: string): void {
     const state = this.socketStates.get(socket);
-    if (!state) {
+    if (!state || this.closing) {
       return;
     }
     if (state.revoked) {
@@ -420,6 +431,7 @@ export class SignalingServer {
             sessionId: state.sessionId,
           },
           () =>
+            !this.closing &&
             this.socketStates.get(socket) === state &&
             !state.revoked &&
             !state.authenticated &&
@@ -462,6 +474,20 @@ export class SignalingServer {
       return;
     }
 
+    if (
+      this.closing ||
+      this.socketStates.get(socket) !== state ||
+      state.revoked ||
+      socket.readyState !== WebSocket.OPEN
+    ) {
+      this.options.roomStore.disconnectParticipant(
+        participant.roomId,
+        participant.peerId,
+        state.sessionId,
+      );
+      return;
+    }
+
     clearTimeout(state.authenticationTimer);
     this.unauthenticatedConnections -= 1;
     let shareGeneration: string | null = null;