From 1b14306b33263354cdcde6b388c762d58243829e Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Mon, 31 Aug 2026 14:46:43 +0800 Subject: [PATCH] perf(media): optimize browser media pipeline evidence (#359) Measured Browser-pipeline improvements, bounded SFU observability, and one-commit release ownership. Local repository hygiene, 860 tests, client/server builds, and the 20-Viewer seven-check gate passed. GitHub Actions did not start because of the account billing/spending-limit annotation. --- CONTRIBUTING.md | 10 +- docs/deployment.md | 10 +- docs/project-memory.md | 7 +- docs/research/realtime-quality-adaptation.md | 146 +++++++- docs/status.md | 42 +-- docs/todo.md | 10 +- scripts/peer-assisted-benchmark.ts | 341 ++++++++++++++++++- src/client/media/sender-quality-evidence.ts | 2 + src/client/pages/HostPage.tsx | 24 +- src/client/sfu/publisher.ts | 8 + src/client/types.ts | 4 + src/client/webrtc/video-codec-preflight.ts | 62 +++- src/server/hybrid-media-router.ts | 4 + src/shared/protocol.ts | 6 + tests/peer-assisted-benchmark.test.ts | 88 +++++ tests/sfu-client.test.ts | 16 + 16 files changed, 714 insertions(+), 66 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index b039406..9fc01dd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,9 +19,15 @@ Use this path only when the owner explicitly requests it or when closing a major 2. Update only the single durable owner for changed semantics and any materially changed current snapshot. If semantics remain disputed, record a hold and stop dependent work. 3. Implement and run repository hygiene, `npm run check`, the relevant browser/network gates, and review in proportion to the whole acceptance boundary. 4. Merge an accepted truth checkpoint before dependent candidates. Rebase or rebuild a retained candidate from that exact `main` once, preserving main's owning truth on conflicts and transplanting only approved scoped code, tests, and new facts. -5. Open the pull request, resolve required review/checks, merge, deploy when authorized, perform scoped postflight, then audit references and reparse safety before cleanup. +5. Open the pull request, resolve required review/checks, squash-merge the coherent phase once, deploy that merged revision when authorized, perform scoped postflight, then audit references and reparse safety before cleanup. -Prefer squash merge for one coherent change and preserve separate commits only when they carry independently useful history. +One coherent phase should leave one meaningful squash commit on `main`. Include +its source, tests, owned semantic documentation, and materially changed status +snapshot in that boundary. Do not open a follow-up pull request whose only +purpose is to copy the deployed revision, asset name, or artifact hashes into +the repository. Exact deployment identity belongs to the immutable release +descriptor, runtime `REVISION`, and deployment record. Preserve separate commits +only when they carry independently useful history. Direct commits to `main` are reserved for an explicit user-approved exception. Never force-push shared branches or rewrite shared history without explicit approval. diff --git a/docs/deployment.md b/docs/deployment.md index 685fc51..8bdb230 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -2,8 +2,10 @@ This file owns routine immutable application releases. Initial service setup is in [self-hosting operations](./operations/self-hosting.md); environment and ports -are in [configuration reference](./reference/configuration.md); exact current -production identity is owned only by [status](./status.md). +are in [configuration reference](./reference/configuration.md). Exact production +identity is owned by the immutable release descriptor, runtime `REVISION`, and +deployment record. [Status](./status.md) owns only the compact current product +and operational snapshot; it is not a per-release ledger. The tracked `release-app.sh` is the updater for the current bare-metal nginx, LiveKit, coturn, nftables, and local-port-8787 deployment. It requires an @@ -41,6 +43,10 @@ full revision, emits a runtime archive plus path/size/SHA-256 manifest and relea extracts its own artifact to verify it. Upload the archive, manifest, and descriptor together to `/opt/screener/uploads`. +Retain the descriptor and successful deployment output as release metadata. Do +not create a follow-up source commit solely to duplicate their revision, asset, +or hashes. + Do not build or run the full repository check on a constrained production host. The release wrapper installs only production dependencies in a transient, CPU/memory/time-bounded unit. diff --git a/docs/project-memory.md b/docs/project-memory.md index 17cda0e..dfda9d6 100644 --- a/docs/project-memory.md +++ b/docs/project-memory.md @@ -58,12 +58,13 @@ implementation and routine UI detail. Canonical source uses the strict `screener-v17` Browser/server contract with committed media readiness, bounded candidate-relative progress and one-shot Host-root convergence. It retains optional SQLite room authority and the -H.264/VP8 sender gate. [Status](./status.md) owns exact source and production -identity. +H.264/VP8 sender gate. [Status](./status.md) owns the compact current execution +snapshot rather than duplicating per-release identity. Current work is owned by the [TODO ledger](./todo.md). Open physical evidence is owned by [verification status](./verification-status.md). Environment and initial services are owned by [configuration](./reference/configuration.md) and [self-hosting](./operations/self-hosting.md); application release is owned by -[deployment](./deployment.md). +[deployment](./deployment.md), with exact deployed identity retained in the +release descriptor and deployment record. Git history and pull requests own completed timelines. diff --git a/docs/research/realtime-quality-adaptation.md b/docs/research/realtime-quality-adaptation.md index 9741434..b05a626 100644 --- a/docs/research/realtime-quality-adaptation.md +++ b/docs/research/realtime-quality-adaptation.md @@ -173,6 +173,26 @@ their first child. The Host's one SFU publication uses the same resolved codec and disables backup codec. A static real capture cannot make the decision inconclusive because probe motion is independent. +Chrome 151.0.7922.175 synthetic 1080p30 startup screening used a fresh process +for each arm. Manual VP8 sent Host authentication 99-116 ms after the share +click. Auto took 3088-3127 ms and correctly fell back to VP8 in all three arms. +Replacing the probe's serial wait-for-complete ICE exchange with standard local +Trickle ICE retained the same codec outcome and cadence proof while repeated +Auto arms fell to 2764-2791 ms, roughly 0.33 seconds faster. The remaining +roughly 2.8 seconds is dominated by full-target warmup and measurement; it is +not grounds to shorten the proof window or silently weaken Auto. + +The same machine also validates why the gate measures delivered cadence instead +of trusting codec support or `qualityLimitationReason`. With forced H.264 at +1080p30, one sender received a roughly 30 fps media source and 6.1-8.6 Mbps of +estimated outgoing bandwidth but encoded only 13-19 fps, averaging 15 fps, while +Chromium reported `none` for every limitation sample. In the three-Viewer arm, +the two Host H.264 senders averaged 14.71 fps; the otherwise equivalent VP8 arm +averaged 28.64 fps at the same 1920x1080 output. The stats exposed neither an +encoder implementation nor a power-efficiency flag, so the result proves only +that this current Browser/device/profile H.264 path is unsuitable. It does not +support a GPU-vendor rule or a hardware/software inference. + VP9, AV1, and H.265 remain unsupported product candidates: Browser codec support does not prove the desired hardware profile, cross-Browser relay compatibility, or better real-time cadence. Screego's VP9 default was reverted after a frame- @@ -210,6 +230,22 @@ default screen-share publication can expose original and lower representations, and server BWE can forward a lower representation to a constrained subscriber while another receives the highest available representation. +Calling the pinned client's own encoding calculator with Screener's exact +publish options produces two VP8 encodings for every accepted profile: + +| Profile | Lower representation | Original representation | +| --- | --- | --- | +| 720p30 | 1/2 scale, 750 kbps, 30 fps | 3 Mbps, 30 fps | +| 1080p30 | 1/2 scale, 1.25 Mbps, 30 fps | 5 Mbps, 30 fps | +| 1080p60 | 1/2 scale, 2 Mbps, 60 fps | 8 Mbps, 60 fps | + +Both encodings start without an explicit RTP `priority` because Screener's +custom `screenShareEncoding` replaces LiveKit's priority-bearing preset while +retaining its default `simulcast: true`. The audio encoding is `high` priority. +Priority can only redistribute bandwidth within that PeerConnection and request +packet marking; it cannot reserve encoder CPU or coordinate independent P2P +connections. This is an A/B input, not evidence to raise video priority. + Forcing a lower encoding permanently consumed the same Host-to-SFU congestion budget and reduced HIGH performance. Publishing only HIGH abandoned constrained subscribers. Current ownership is therefore: @@ -241,9 +277,12 @@ warning-level logs contain no subscribed-quality or publisher-layer transition, so Dynacast disablement is not established as the cause of each freeze. SFU publisher diagnostics therefore keep aggregate bitrate and native limitation across all active representations while reporting dimensions and cadence from -the highest active representation. A future event can distinguish publisher -layer disablement from subscriber-only forwarding changes without altering the -LiveKit adaptation policy. +the highest active representation. They also report the total and currently +active sender-encoding counts read from the existing publisher parameters. The +counts contain no RID or participant identity and do not affect routing or layer +selection. A future event can therefore distinguish publisher layer disablement +from subscriber-only forwarding changes without altering the LiveKit adaptation +policy. ## Host And Page Cost Boundaries @@ -259,6 +298,15 @@ mobile suspension, page reclamation, and relay survival remain platform evidence in [background capture research](./browser-background-capture.md), not Web keepalive features. +The Media Capture specification copies a track's constraints when it is cloned, +then lets each clone change constraints independently. Current Host Peer senders +trust those copied constraints when constructed, while the SFU publisher applies +the same profile again immediately after creating an equivalent clone. Removing +only that initial duplicate application should preserve constraints; clearing +clone constraints altogether would change the source/sink contract and requires +an A/B across live profile changes, sibling pressure, source replacement and +same-edge regeneration. + Web content has no supported API for raising capture, encoder, renderer-process, GPU, or operating-system scheduling priority. WebRTC sender `priority` allocates bandwidth relative to other RTP senders and `networkPriority` requests DSCP; @@ -272,6 +320,89 @@ publication when its accepted route condition applies, or a future native/shared encoder. LiveKit Dynacast can stop unused SFU representations; it cannot combine independent P2P encoders. +### Current Pipeline Screening + +Chrome 151.0.7922.175 headless loopbacks on 2026-08-31 used one isolated +Browser process, deterministic high-motion canvas capture and three Viewer +pages on the same machine. They exercise real capture tracks, PeerConnections, +VP8 encode/decode and the deployed Peer topology, but deliberately amplify +shared Browser/GPU/CPU contention and are not a distributed-network claim. + +| Profile and topology | Host encode | Host cadence | Browser CPU | Receive result | +| --- | ---: | ---: | ---: | --- | +| 720p30, 3 Viewers | 3.62 ms/frame | 29.93 fps | 163% average | 30.06 fps; no freeze | +| 1080p30, 3 Viewers, run 1 | 10.83 ms/frame | 29.25 fps | 353% average | 28.87 fps; no freeze | +| 1080p30, 3 Viewers, run 2 | 11.00 ms/frame | 28.92 fps | 374% average | 28.53 fps; no freeze | +| 1080p30, 3 Viewers, run 3 | 19.71 ms/frame | 25.25 fps | 450% average | 25.40 fps; 7 freezes / 3.38 s | + +Each repeat launched a fresh Chrome process. The spread proves a resource cliff +and run-to-run system sensitivity, not a deterministic three-Viewer limit. A +prior 1-to-2-to-3 sequence in one Browser process produced a similarly poor +last arm even though every page was closed between cases; cross-arm Browser +state and a 250 ms cleanup gap make that sequence unsuitable for performance +comparison. Future arms must use independent processes, repeated/randomized +order, actual output resolution, and system-load context. Aggregate +`qualityLimitationReason` alone did not describe the worst arm. + +The same harness then added per-page CDP `TaskDuration`, script, layout, style +and heap deltas using each page's own monotonic timestamp. In a 20-Viewer +720p30 burst, all route checks passed and every Viewer decoded, but the Host +page used 36.35% of one main thread on average and briefly approached a full +thread; almost all measured work was script, not layout or style. This validates +the existing Host-only diagnostic-rendering gate without attributing the +same-machine media CPU to production. + +The accepted implementation keeps every quality-evidence ref update and +freshness timer immediate while coalescing only the React presentation commit +to at most one animation frame. Two independent after-runs retained all seven +route checks and every Viewer decode while Host main-thread task utilization was +20.89% and 25.09%, with a 31.41% maximum interval. The change adds no media +sampling interval, route delay or quality threshold. + +A final 20-Viewer acceptance run sampled the Host plus one plain Viewer and one +relay while retaining full-page topology and decode checks. All seven checks +passed, every Viewer decoded, and Host task utilization was 22.22% average and +34.48% peak. The isolated same-machine Browser consumed 738% aggregate CPU and +Host media-source cadence averaged 14.93 fps, so this arm validates bounded +topology and diagnostic rendering rather than representative distributed media +headroom. + +The current stage decisions are therefore deliberately narrow: + +| Stage | Current owner and decision | +| --- | --- | +| Capture | Keep native `getDisplayMedia`, profile constraints and `motion`; no JS video preprocessing or synthetic keepalive. | +| Codec startup | Keep the real full-profile cadence gate; use local Trickle ICE, but do not shorten its proof or cache a device-wide verdict. | +| Peer and relay encode | Keep one isolated clone and stock WebRTC adaptation per sender under the existing endpoint cap. Browser relay still decodes and re-encodes. | +| SFU publication | Keep pinned LiveKit's two representations, Dynacast and send-side BWE. Observe total/active encoding counts before changing layer policy. | +| Receive latency | Keep the Browser jitter buffer and A/V synchronizer; no fixed `jitterBufferTarget` without a loss/latency A/B. | +| Presentation | Coalesce only Host diagnostic React commits. Media evidence, freshness and route control stay immediate. | +| Native boundary | OS capture fallback, virtual display, shared/zero-copy encode and driver-specific capability caching require a separately accepted native product surface. | + +### Mature Product Boundaries + +Discord documents a multi-process capture/encode/transport/decode pipeline, +OS-specific capture fallback, hardware codec selection, WebRTC bandwidth +estimation, and joint monitoring of cadence, latency, visual quality, network, +CPU and memory. Its desktop client uses native capture and codec integration; +that result is evidence for a future native boundary, not a Browser API recipe. +Discord also reported a real frame-drop ratchet where reconfiguring an encoder +to its already-reduced cadence increased bits per frame and compounded drops. + +A signed local NetEase UU Remote 4.38.3 installation separates UI, service, +streamer, codec detector, virtual-display and audio components. Its non-sensitive +capability caches contain 41 encoder and 74 decoder entries keyed by codec, +adapter/device, dimensions, frame rate, implementation, chroma sampling and bit +depth. This supports profile-scoped capability caching and process isolation as +mature native practices; enum meanings and proprietary algorithms were not +inferred, and the detector was not executed. + +KOOK's official 1080p60 guidance requires substantial CPU/GPU headroom and its +troubleshooting explicitly notes that hardware acceleration can worsen some +systems. Oopz treats VPN/accelerator changes as an independent network-path +diagnostic. Together these support measured fallback and stage-specific +diagnosis rather than a universal hardware, codec or transport switch. + Production room 4521 supplied a separate network/source boundary. It used one stable Host-to-Viewer P2P edge with no reparent or SFU activity, yet Chromium reported `bandwidth` for 112 consecutive sender windows, estimated only @@ -394,6 +525,10 @@ percentages, and tuning loops are not adopted for this Browser product. - Controlled and production correlation of Host SFU active representations, subscribed-quality changes, keyframe reacquisition, and Viewer freezes before changing Dynacast policy or its delay. +- Current-Browser A/B of the duplicate initial SFU clone constraint application, + source-only versus independently constrained clones across live profile + changes, and default two-layer screen-share publication versus any proposed + representation or priority change. ## Primary Sources @@ -418,6 +553,11 @@ percentages, and tuning loops are not adopted for this Browser product. - [Picture-in-Picture](https://www.w3.org/TR/picture-in-picture/) - [Chrome timer throttling](https://developer.chrome.com/blog/timer-throttling-in-chrome-88) - [Chrome Page Lifecycle](https://developer.chrome.com/docs/web-platform/page-lifecycle-api) +- [Discord Go Live pipeline](https://discord.com/blog/how-it-all-goes-live-an-overview-of-discords-streaming-technology) +- [Discord AMD encoder ratchet](https://discord.com/blog/from-blocky-to-brilliant-improving-video-quality-on-discord-go-live-on-amd-gpus) +- [KOOK screen-share requirements](https://support.kookapp.cn/7ff3/b671) +- [KOOK resource troubleshooting](https://help.kookapp.cn/6a2f/8525) +- [Oopz network-path troubleshooting](https://help.oopz.cn/fa71/694b) - [LiveKit server forwarder](https://github.com/livekit/livekit/blob/v1.13.5/pkg/sfu/forwarder.go) - [LiveKit server defaults](https://github.com/livekit/livekit/blob/v1.13.5/pkg/config/config.go) - [LiveKit Dynacast manager](https://github.com/livekit/livekit/blob/v1.13.5/pkg/rtc/dynacast/dynacastmanagervideo.go) diff --git a/docs/status.md b/docs/status.md index 472b998..570d8f4 100644 --- a/docs/status.md +++ b/docs/status.md @@ -7,20 +7,13 @@ This is the current execution index. Git history owns completed timelines; ## Production -- `https://share.bonfire.icu` runs exact application/runtime revision - `718b30cd91a505918ea772ed0dd8075c6caa4232`, release `718b30c`, wire - `screener-v17`, from `/opt/screener/releases/718b30c`. The immutable runtime - tar SHA-256 is - `672dd8a4573290b9201e3d09f1350a46cea3dcc9861b4ad825ecf5bf5f3c7a49`; - its 41-file manifest SHA-256 is - `f078a28c5440882dfbca40dcd714ca65c21630cd57657ee7567f562a9373a8cf`. -- The served Browser entry references `assets/index-Bsb-88DT.js`; the public - asset SHA-256 is - `2190502fd8091efad910690f692658c750ed1088901e39052303016d1949d444`. - Public `/healthz` returns 200. The release postflight found Screener, LiveKit, - coturn, and nginx active with zero restarts. Controlled application restarts - have retained SQLite room authority and Host capture while P2P/SFU routes and - Viewer diagnostics recovered without a page refresh. +- `https://share.bonfire.icu` runs the strict `screener-v17` Browser/server + contract. Exact revision, release, artifact, manifest, and asset identity are + retained by the immutable release descriptor, runtime `REVISION`, and + deployment record rather than copied into this source snapshot. +- The latest scoped postflight found public health and the immutable Browser + asset available, with Screener, LiveKit, coturn, and nginx active and without + restarts. - Production enables SQLite room authority at `/var/lib/screener/rooms.sqlite`; live participants, routes and media remain process-only. A controlled restart retained the room authority and Host-owned @@ -105,27 +98,22 @@ This is the current execution index. Git history owns completed timelines; Host and Viewer couch/topology identities stay consistent across expression modes, and bounded deep trees scroll within their own surface. -## Source/Production Delta +## Source/Production Relationship -- Production application code matches canonical source revision - `718b30cd91a505918ea772ed0dd8075c6caa4232` exactly. +- The deployment record owns exact source/release comparison. Routine releases + change this file only when its semantic or operational snapshot changes. ## Active Boundaries -- Desktop Host background capture remains unconfirmed. The current-browser - screening did not reproduce a Host-page drop; accepted Viewer lifecycle work - prevents frozen JavaScript wall time from becoming an immediate route failure - but is not capture keepalive. +- Desktop Host background capture remains unconfirmed. Current Browser screening + did not reproduce a drop; Viewer lifecycle work prevents frozen JavaScript + wall time from becoming route-failure evidence but is not capture keepalive. - Current-path quality can drive local convergence in source and production, including bounded same-edge connection regeneration. Both default it on with a pre-share Host opt-out. Weighted/global optimization remains parked. -- Repository simplification does not change or deploy product behavior beyond - deleting proven dead private surfaces. - Native/executable senders, shared encode, and distribution packages remain - outside the current release. Current UI refinement is presentation-only and - does not authorize route or media-model changes. + outside the current release. ## Current Hold -No source or deployment P0/P1 is open. Sanitized route logging remains enabled -for the pre-release canary. +No source or deployment P0/P1 is open; sanitized route logging remains enabled. diff --git a/docs/todo.md b/docs/todo.md index 69ebefe..cd4d8d9 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -28,10 +28,10 @@ until their stated boundary is met. prove a self-hosted stale-token revocation boundary before independently releasing subscription egress; do not introduce accounts, polling, risk scoring, or an independent policy framework. -4. **Client ownership and diagnostic rendering.** Batch Host-only diagnostic - snapshots when a 20-Viewer measurement proves rendering cost, and extract - media-session owners from the Host/Viewer pages only alongside reachable - behavior work. File length alone does not authorize a page rewrite. +4. **Client ownership.** Extract media-session owners from the Host/Viewer pages + only alongside reachable behavior work. File length alone does not authorize + a page rewrite; Host diagnostic presentation is already coalesced without + delaying media evidence or route control. 5. **C=3 structural intent retention.** Revisit multiple simultaneous one-shot Host-root convergence intents only with an exact C=3 reproduction; do not add periodic balancing or another topology queue for the default path. @@ -41,6 +41,8 @@ until their stated boundary is met. 7. **Representative network acceptance.** Complete public-network direct, Browser relay, SFU, recovery, Pause/Resume, screen-audio, real-game A/V, two-room SFU, 20-Viewer endurance, and all-UDP-blocked bounded failure. + Correlate SFU freezes with publisher encoding counts before changing + Dynacast or representation policy. 8. **Client-input security review.** Audit HTTP/WebSocket schema, auth, authorization, rate/body/resource bounds, errors, logs, and secrets without adding a parallel security framework. diff --git a/scripts/peer-assisted-benchmark.ts b/scripts/peer-assisted-benchmark.ts index 19f2bd4..89226ea 100644 --- a/scripts/peer-assisted-benchmark.ts +++ b/scripts/peer-assisted-benchmark.ts @@ -34,6 +34,7 @@ import { const PROFILE_SETTINGS = QUALITY_PROFILES; type ProfileId = QualityProfileId; type PageRole = "host" | "viewer"; +export type BenchmarkCodecMode = "vp8" | "auto" | "h264"; export type BenchmarkCanaryMode = "none" | "viewer-mbb"; const ROUTE_TIMING_KEYS = [ @@ -74,6 +75,7 @@ export interface BenchmarkConfig { viewerCounts: number[]; expectedEndpointCap: number; profileId: ProfileId; + codecMode: BenchmarkCodecMode; durationMs: number; settleMs: number; sampleIntervalMs: number; @@ -109,6 +111,15 @@ interface ConnectionObservation { error?: string; } +export interface PagePerformanceSample { + timestampSeconds: number; + taskDurationSeconds: number; + scriptDurationSeconds: number; + layoutDurationSeconds: number; + recalcStyleDurationSeconds: number; + jsHeapUsedBytes: number; +} + interface PageObservation { label: string; role: PageRole; @@ -117,6 +128,7 @@ interface PageObservation { peerId: string | null; authenticatedAtEpochMs: number | null; authenticateSentAtEpochMs: number | null; + shareRequestedAtEpochMs?: number | null; signalingConnected: boolean; qualitySettings: QualitySettings | null; routeRevision: number | null; @@ -127,6 +139,7 @@ interface PageObservation { firstRenderedAtEpochMs: number | null; renderedFrames: number; connections: ConnectionObservation[]; + performance?: PagePerformanceSample | null; } interface FailurePageEvidence { @@ -217,7 +230,7 @@ interface BenchmarkRun { } interface BenchmarkReport { - schemaVersion: 3; + schemaVersion: 4; startedAt: string; completedAt: string | null; gitCommit: string | null; @@ -563,6 +576,14 @@ export function parseBenchmarkCanaryMode(value: string | undefined): BenchmarkCa throw new Error("BENCHMARK_CANARY must be none or viewer-mbb"); } +export function parseBenchmarkCodecMode( + value: string | undefined, +): BenchmarkCodecMode { + const mode = value?.trim().toLowerCase() || "auto"; + if (mode === "vp8" || mode === "auto" || mode === "h264") return mode; + throw new Error("BENCHMARK_CODEC_MODE must be vp8, auto, or h264"); +} + export function parseExpectedEndpointCap(value: string | undefined): number { const parsed = value?.trim() ? Number(value) @@ -626,6 +647,7 @@ export function parseBenchmarkConfig( environment.BENCHMARK_EXPECTED_ENDPOINT_CAP, ), profileId, + codecMode: parseBenchmarkCodecMode(environment.BENCHMARK_CODEC_MODE), durationMs: parseNumber( environment.BENCHMARK_DURATION_SECONDS, @@ -828,6 +850,51 @@ function inspectSfuPublication(pages: readonly PageObservation[]) { return { observed: true, rootCount: sfuViewers.length, coherent }; } +const PAGE_PERFORMANCE_METRICS = { + Timestamp: "timestampSeconds", + TaskDuration: "taskDurationSeconds", + ScriptDuration: "scriptDurationSeconds", + LayoutDuration: "layoutDurationSeconds", + RecalcStyleDuration: "recalcStyleDurationSeconds", + JSHeapUsedSize: "jsHeapUsedBytes", +} as const satisfies Record; + +export function parsePagePerformanceMetrics( + rawMetrics: unknown, +): PagePerformanceSample | null { + if (!Array.isArray(rawMetrics)) return null; + const values = new Map(); + for (const raw of rawMetrics) { + if (!raw || typeof raw !== "object") continue; + const { name, value } = raw as Record; + if ( + typeof name !== "string" || + !Object.hasOwn(PAGE_PERFORMANCE_METRICS, name) + ) { + continue; + } + if (typeof value !== "number" || !Number.isFinite(value) || value < 0) { + return null; + } + const key = PAGE_PERFORMANCE_METRICS[ + name as keyof typeof PAGE_PERFORMANCE_METRICS + ]; + if (values.has(key)) return null; + values.set(key, value); + } + if (values.size !== Object.keys(PAGE_PERFORMANCE_METRICS).length) { + return null; + } + return { + timestampSeconds: values.get("timestampSeconds")!, + taskDurationSeconds: values.get("taskDurationSeconds")!, + scriptDurationSeconds: values.get("scriptDurationSeconds")!, + layoutDurationSeconds: values.get("layoutDurationSeconds")!, + recalcStyleDurationSeconds: values.get("recalcStyleDurationSeconds")!, + jsHeapUsedBytes: values.get("jsHeapUsedBytes")!, + }; +} + export function summarizeSamples(samples: TimedSample[], viewerCount: number) { let maxHostActiveMediaEdges = 0; let maxHostAssignedChildren = 0; @@ -939,6 +1006,14 @@ export function summarizeSamples(samples: TimedSample[], viewerCount: number) { ? [] : [frame.decodedAfterAuthenticateMs], ); + const finalHost = finalPages.find((page) => page.role === "host"); + const shareToAuthenticateMs = + finalHost?.shareRequestedAtEpochMs !== null && + finalHost?.shareRequestedAtEpochMs !== undefined && + finalHost.authenticateSentAtEpochMs !== null + ? finalHost.authenticateSentAtEpochMs - + finalHost.shareRequestedAtEpochMs + : null; return { maxHostActiveMediaEdges, maxHostAssignedChildren, @@ -951,12 +1026,16 @@ export function summarizeSamples(samples: TimedSample[], viewerCount: number) { firstFrames, maxFirstDecodedAfterAuthenticateMs: decodedDelays.length === viewerCount ? Math.max(...decodedDelays) : null, + hostStartup: { + shareToAuthenticateMs, + }, senderEvidence: { scope: "connected outbound video with stable page, connection, and RTP identities", host: summarizeSenderEvidence(samples, "host"), relay: summarizeSenderEvidence(samples, "viewer"), }, browserProcessResources: summarizeBrowserProcessResources(samples), + pageRuntimeResources: summarizePageRuntimeResources(samples), finalTopology: finalPages.map((page) => ({ label: page.label, role: page.role, @@ -1007,11 +1086,19 @@ function numericSummary(values: number[]) { function summarizeSenderEvidence(samples: TimedSample[], role: PageRole) { const identities = new Set(); const bitrateKbps: number[] = [], framesPerSecond: number[] = []; + const captureFramesPerSecond: number[] = []; + const mediaSourceFramesPerSecond: number[] = []; const availableOutgoingKbps: number[] = []; const resolutions = new Set(); + const codecs = new Set(); + const encoderImplementations = new Set(); + const scalabilityModes = new Set(); const qualityLimitationReasonSamples: Record = {}; let unknownIdentitySamples = 0; let unknownQualityLimitationSamples = 0; + let powerEfficientEncoderTrueSamples = 0; + let powerEfficientEncoderFalseSamples = 0; + let powerEfficientEncoderUnknownSamples = 0; let encodeIntervalCount = 0, framesEncoded = 0, encodeTimeMs = 0; const finite = (value: unknown): value is number => typeof value === "number" && Number.isFinite(value); @@ -1028,8 +1115,20 @@ function summarizeSenderEvidence(samples: TimedSample[], role: PageRole) { identities.add(`${page.label}:${connection.createdAtEpochMs}:${connection.connectionId ?? connection.index}:${rtpStatsId}`); if (finite(connection.send.bitrateKbps)) bitrateKbps.push(connection.send.bitrateKbps); if (finite(connection.send.framesPerSecond)) framesPerSecond.push(connection.send.framesPerSecond); + if (finite(connection.send.captureFramesPerSecond)) captureFramesPerSecond.push(connection.send.captureFramesPerSecond); + if (finite(connection.send.mediaSourceFramesPerSecond)) mediaSourceFramesPerSecond.push(connection.send.mediaSourceFramesPerSecond); if (finite(connection.send.availableOutgoingKbps)) availableOutgoingKbps.push(connection.send.availableOutgoingKbps); if (typeof connection.send.resolution === "string" && /^\d+x\d+$/.test(connection.send.resolution)) resolutions.add(connection.send.resolution); + if (typeof connection.send.codec === "string" && connection.send.codec.length > 0) codecs.add(connection.send.codec); + if (typeof connection.send.encoderImplementation === "string" && connection.send.encoderImplementation.length > 0) encoderImplementations.add(connection.send.encoderImplementation); + if (typeof connection.send.scalabilityMode === "string" && connection.send.scalabilityMode.length > 0) scalabilityModes.add(connection.send.scalabilityMode); + if (connection.send.powerEfficientEncoder === true) { + powerEfficientEncoderTrueSamples += 1; + } else if (connection.send.powerEfficientEncoder === false) { + powerEfficientEncoderFalseSamples += 1; + } else { + powerEfficientEncoderUnknownSamples += 1; + } const reason = connection.send.qualityLimitationReason; if (typeof reason === "string" && reason.length > 0) { qualityLimitationReasonSamples[reason] = (qualityLimitationReasonSamples[reason] ?? 0) + 1; @@ -1051,7 +1150,21 @@ function summarizeSenderEvidence(samples: TimedSample[], role: PageRole) { unknownIdentitySamples, bitrateKbps: numericSummary(bitrateKbps), framesPerSecond: numericSummary(framesPerSecond), + captureFramesPerSecond: numericSummary(captureFramesPerSecond), + mediaSourceFramesPerSecond: numericSummary(mediaSourceFramesPerSecond), resolutions: resolutions.size > 0 ? [...resolutions].sort() : null, + codecs: codecs.size > 0 ? [...codecs].sort() : null, + encoderImplementations: + encoderImplementations.size > 0 + ? [...encoderImplementations].sort() + : null, + scalabilityModes: + scalabilityModes.size > 0 ? [...scalabilityModes].sort() : null, + powerEfficientEncoderSamples: { + true: powerEfficientEncoderTrueSamples, + false: powerEfficientEncoderFalseSamples, + unknown: powerEfficientEncoderUnknownSamples, + }, availableOutgoingKbps: numericSummary(availableOutgoingKbps), encodeIntervals: { sampleCount: encodeIntervalCount, @@ -1109,6 +1222,141 @@ function summarizeBrowserProcessResources(samples: TimedSample[]) { }; } +type PageRuntimeRole = "host" | "viewer" | "relay"; + +interface PageRuntimeAccumulator { + validIntervals: number; + invalidIntervals: number; + wallSeconds: number; + taskSeconds: number; + scriptSeconds: number; + layoutSeconds: number; + recalcStyleSeconds: number; + peakTaskUtilizationPercent: number | null; + peakJsHeapUsedBytes: number | null; +} + +function pageRuntimeRole(page: PageObservation): PageRuntimeRole { + if (page.role === "host") return "host"; + return activeVideoEdgeCount(page, "send") > 0 ? "relay" : "viewer"; +} + +function emptyPageRuntimeAccumulator(): PageRuntimeAccumulator { + return { + validIntervals: 0, + invalidIntervals: 0, + wallSeconds: 0, + taskSeconds: 0, + scriptSeconds: 0, + layoutSeconds: 0, + recalcStyleSeconds: 0, + peakTaskUtilizationPercent: null, + peakJsHeapUsedBytes: null, + }; +} + +function finishPageRuntime(accumulator: PageRuntimeAccumulator) { + return { + validIntervals: accumulator.validIntervals, + invalidIntervals: accumulator.invalidIntervals, + measuredWallTimeSeconds: + accumulator.validIntervals > 0 ? accumulator.wallSeconds : null, + taskTimeSeconds: + accumulator.validIntervals > 0 ? accumulator.taskSeconds : null, + averageTaskUtilizationPercent: + accumulator.wallSeconds > 0 + ? (accumulator.taskSeconds / accumulator.wallSeconds) * 100 + : null, + peakTaskUtilizationPercent: accumulator.peakTaskUtilizationPercent, + scriptTimeSeconds: + accumulator.validIntervals > 0 ? accumulator.scriptSeconds : null, + layoutTimeSeconds: + accumulator.validIntervals > 0 ? accumulator.layoutSeconds : null, + recalcStyleTimeSeconds: + accumulator.validIntervals > 0 + ? accumulator.recalcStyleSeconds + : null, + peakJsHeapUsedBytes: accumulator.peakJsHeapUsedBytes, + }; +} + +export function summarizePageRuntimeResources(samples: TimedSample[]) { + const accumulators: Record = { + host: emptyPageRuntimeAccumulator(), + viewer: emptyPageRuntimeAccumulator(), + relay: emptyPageRuntimeAccumulator(), + }; + for (const sample of samples) { + for (const page of sample.pages) { + const heap = page.performance?.jsHeapUsedBytes; + if (heap === undefined) continue; + const accumulator = accumulators[pageRuntimeRole(page)]; + accumulator.peakJsHeapUsedBytes = Math.max( + accumulator.peakJsHeapUsedBytes ?? 0, + heap, + ); + } + } + for (let index = 1; index < samples.length; index += 1) { + const previous = samples[index - 1]!; + const current = samples[index]!; + const previousPages = new Map( + previous.pages.map((page) => [page.label, page]), + ); + for (const page of current.pages) { + const role = pageRuntimeRole(page); + const accumulator = accumulators[role]; + const before = previousPages.get(page.label)?.performance; + const after = page.performance; + if (!before && !after) continue; + const wallSeconds = + before && after + ? after.timestampSeconds - before.timestampSeconds + : null; + const deltas = before && after + ? { + task: after.taskDurationSeconds - before.taskDurationSeconds, + script: after.scriptDurationSeconds - before.scriptDurationSeconds, + layout: after.layoutDurationSeconds - before.layoutDurationSeconds, + recalc: + after.recalcStyleDurationSeconds - + before.recalcStyleDurationSeconds, + } + : null; + if ( + wallSeconds === null || + !Number.isFinite(wallSeconds) || + wallSeconds <= 0 || + !deltas || + Object.values(deltas).some( + (value) => !Number.isFinite(value) || value < 0, + ) + ) { + accumulator.invalidIntervals += 1; + continue; + } + accumulator.validIntervals += 1; + accumulator.wallSeconds += wallSeconds; + accumulator.taskSeconds += deltas.task; + accumulator.scriptSeconds += deltas.script; + accumulator.layoutSeconds += deltas.layout; + accumulator.recalcStyleSeconds += deltas.recalc; + accumulator.peakTaskUtilizationPercent = Math.max( + accumulator.peakTaskUtilizationPercent ?? 0, + (deltas.task / wallSeconds) * 100, + ); + } + } + return { + source: "CDP Performance.getMetrics", + scope: + "Host plus one representative plain Viewer and relay per sample; relay means a Viewer page with active outbound video", + host: finishPageRuntime(accumulators.host), + viewer: finishPageRuntime(accumulators.viewer), + relay: finishPageRuntime(accumulators.relay), + }; +} + export function buildRunChecks( summary: NonNullable, viewerCount: number, @@ -1218,6 +1466,7 @@ export function buildBenchmarkInitScript(options: { peerId: null, authenticatedAtEpochMs: null, authenticateSentAtEpochMs: null, + shareRequestedAtEpochMs: null, signalingConnected: false, qualitySettings: null, routeRevision: null, @@ -1735,9 +1984,12 @@ export function buildBenchmarkInitScript(options: { function sendViewerQualityEvidence(message) { return message?.type === "viewer-quality-evidence" && sendCanaryMessage(message); } + function markShareRequested() { + state.shareRequestedAtEpochMs = Date.now(); + } Object.defineProperty(globalThis, "__SCREENER_BENCHMARK__", { configurable: false, - value: { sample, progress, snapshot, canarySnapshot, sendViewerQualityEvidence, requestRouteDiagnosticSnapshot, routeDiagnosticTimingSamples, stop: () => clearInterval(videoObserver) }, + value: { sample, progress, snapshot, canarySnapshot, sendViewerQualityEvidence, markShareRequested, requestRouteDiagnosticSnapshot, routeDiagnosticTimingSamples, stop: () => clearInterval(videoObserver) }, }); })();`; } @@ -1883,6 +2135,7 @@ export async function createPage( await Promise.all([ cdp.call("Page.enable", {}, page.sessionId), cdp.call("Runtime.enable", {}, page.sessionId), + cdp.call("Performance.enable", {}, page.sessionId), ]); await cdp.call( "Page.addScriptToEvaluateOnNewDocument", @@ -2012,6 +2265,7 @@ async function startHost( cdp: CdpConnection, page: PageHandle, profileId: ProfileId, + codecMode: BenchmarkCodecMode, timeoutMs: number, signal: AbortSignal, ): Promise { @@ -2024,6 +2278,40 @@ async function startHost( "host controls", signal, ); + if (codecMode !== "auto") { + await evaluate( + cdp, + page, + `(() => { + const advanced = document.querySelector('button[aria-controls="host-advanced-door"]'); + if (!(advanced instanceof HTMLButtonElement)) throw new Error('Advanced settings button missing'); + advanced.click(); + return true; + })()`, + ); + await waitForPage( + cdp, + page, + `Array.from(document.querySelectorAll('#host-advanced-door button')).some( + (button) => button instanceof HTMLButtonElement && button.getAttribute('aria-label')?.startsWith('${codecMode.toUpperCase()}') + )`, + 5_000, + `${codecMode} codec control`, + signal, + ); + await evaluate( + cdp, + page, + `(() => { + const button = Array.from(document.querySelectorAll('#host-advanced-door button')).find( + (candidate) => candidate instanceof HTMLButtonElement && candidate.getAttribute('aria-label')?.startsWith('${codecMode.toUpperCase()}') + ); + if (!(button instanceof HTMLButtonElement)) throw new Error('Codec mode button missing'); + button.click(); + return true; + })()`, + ); + } await evaluate( cdp, page, @@ -2034,6 +2322,7 @@ async function startHost( profile.click(); const start = document.querySelector('.lr-entry-actions button.lr-tv-big.is-action[aria-label]'); if (!(start instanceof HTMLButtonElement)) throw new Error('Start button missing'); + globalThis.__SCREENER_BENCHMARK__.markShareRequested(); start.click(); return true; })()`, @@ -2115,10 +2404,30 @@ async function samplePages( startedAtMs: number, ): Promise { const atEpochMs = Date.now(); - const [observations, browserProcesses] = await Promise.all([ - Promise.all(pages.map((page) => detailedSample(cdp, page))), - sampleBrowserProcesses(cdp), - ]); + const browserProcessesPromise = sampleBrowserProcesses(cdp); + const observations: PageObservation[] = []; + let sampledViewerRuntime = false; + let sampledRelayRuntime = false; + for (const page of pages) { + let observation: PageObservation; + try { + observation = await detailedSample(cdp, page); + } catch (error) { + throw new Error(`${page.label} sample failed: ${errorMessage(error)}`); + } + const runtimeRole = pageRuntimeRole(observation); + const sampleRuntime: boolean = + runtimeRole === "host" || + (runtimeRole === "viewer" && !sampledViewerRuntime) || + (runtimeRole === "relay" && !sampledRelayRuntime); + const performance = sampleRuntime + ? await samplePagePerformance(cdp, page) + : null; + sampledViewerRuntime ||= runtimeRole === "viewer" && sampleRuntime; + sampledRelayRuntime ||= runtimeRole === "relay" && sampleRuntime; + observations.push({ ...observation, performance }); + } + const browserProcesses = await browserProcessesPromise; return { atEpochMs, elapsedMs: atEpochMs - startedAtMs, @@ -2127,6 +2436,22 @@ async function samplePages( }; } +async function samplePagePerformance( + cdp: CdpConnection, + page: PageHandle, +): Promise { + try { + const result = await cdp.call<{ metrics: unknown }>( + "Performance.getMetrics", + {}, + page.sessionId, + ); + return parsePagePerformanceMetrics(result.metrics); + } catch { + return null; + } +} + async function sampleBrowserProcesses(cdp: CdpConnection): Promise { try { const result = await cdp.call<{ processInfo: unknown[] }>("SystemInfo.getProcessInfo"); @@ -2753,6 +3078,7 @@ async function runCase( cdp, hostPage, config.profileId, + config.codecMode, config.connectionTimeoutMs, signal, ); @@ -2997,7 +3323,7 @@ export async function main(): Promise { const profile = PROFILE_SETTINGS[config.profileId]; const profileResolution = QUALITY_RESOLUTIONS[profile.resolution]; const report: BenchmarkReport = { - schemaVersion: 3, + schemaVersion: 4, startedAt: new Date().toISOString(), completedAt: null, gitCommit: await gitCommit(), @@ -3012,6 +3338,7 @@ export async function main(): Promise { viewerCounts: config.viewerCounts, expectedEndpointCap: config.expectedEndpointCap, profileId: config.profileId, + codecMode: config.codecMode, durationMs: config.durationMs, settleMs: config.settleMs, sampleIntervalMs: config.sampleIntervalMs, diff --git a/src/client/media/sender-quality-evidence.ts b/src/client/media/sender-quality-evidence.ts index fa79741..c306e46 100644 --- a/src/client/media/sender-quality-evidence.ts +++ b/src/client/media/sender-quality-evidence.ts @@ -173,6 +173,8 @@ export function sfuPublisherQualityEvidenceFromMetrics( mediaSourceFramesPerSecond: metrics.mediaSourceFramesPerSecond, width: metrics.frameWidth, height: metrics.frameHeight, + videoEncodingCount: metrics.videoEncodingCount, + activeVideoEncodingCount: metrics.activeVideoEncodingCount, availableOutgoingKbps: metrics.availableOutgoingKbps, rttMs: metrics.rttMs, packetLossPercent: metrics.packetLossPercent, diff --git a/src/client/pages/HostPage.tsx b/src/client/pages/HostPage.tsx index 7bb53da..44e8dee 100644 --- a/src/client/pages/HostPage.tsx +++ b/src/client/pages/HostPage.tsx @@ -504,6 +504,7 @@ export function HostPage({ onAuthorizationRequired }: HostPageProps = {}) { const viewerQualityEvidenceTimersRef = useRef( new Map(), ); + const viewerQualityEvidenceRenderFrameRef = useRef(null); const peerAssistedRef = useRef(false); const activeRouteRevisionRef = useRef(0); const generationRef = useRef(0); @@ -629,6 +630,7 @@ export function HostPage({ onAuthorizationRequired }: HostPageProps = {}) { ); viewerQualityEvidenceTimersRef.current.clear(); viewerQualityEvidenceRef.current.clear(); + cancelViewerQualityEvidenceRender(); activeRouteRevisionRef.current = 0; void hostSfuRouteRef.current?.disconnect(); hostSfuRouteRef.current = null; @@ -810,6 +812,7 @@ export function HostPage({ onAuthorizationRequired }: HostPageProps = {}) { window.clearTimeout(timer), ); viewerQualityEvidenceTimersRef.current.clear(); + cancelViewerQualityEvidenceRender(); viewerQualityEvidenceRef.current = new Map(); setViewerQualityEvidence(new Map()); activeRouteRevisionRef.current = 0; @@ -1007,7 +1010,7 @@ export function HostPage({ onAuthorizationRequired }: HostPageProps = {}) { next.set(peerId, presentation); } viewerQualityEvidenceRef.current = next; - setViewerQualityEvidence(next); + scheduleViewerQualityEvidenceRender(); } if (presentation === null) { return; @@ -1038,6 +1041,23 @@ export function HostPage({ onAuthorizationRequired }: HostPageProps = {}) { commitViewerQualityEvidence(peerId, null); } + function scheduleViewerQualityEvidenceRender(): void { + if (viewerQualityEvidenceRenderFrameRef.current !== null) return; + viewerQualityEvidenceRenderFrameRef.current = window.requestAnimationFrame( + () => { + viewerQualityEvidenceRenderFrameRef.current = null; + setViewerQualityEvidence(viewerQualityEvidenceRef.current); + }, + ); + } + + function cancelViewerQualityEvidenceRender(): void { + const frame = viewerQualityEvidenceRenderFrameRef.current; + if (frame === null) return; + window.cancelAnimationFrame(frame); + viewerQualityEvidenceRenderFrameRef.current = null; + } + function retainViewerQualityEvidenceForPresence( entries: readonly ParticipantPresenceEntry[], ): void { @@ -1059,7 +1079,7 @@ export function HostPage({ onAuthorizationRequired }: HostPageProps = {}) { viewerQualityEvidenceTimersRef.current.delete(peerId); } viewerQualityEvidenceRef.current = retained; - setViewerQualityEvidence(retained); + scheduleViewerQualityEvidenceRender(); } function acceptViewerQualityEvidence(evidence: ViewerQualityEvidence): void { diff --git a/src/client/sfu/publisher.ts b/src/client/sfu/publisher.ts index be697c5..70849cc 100644 --- a/src/client/sfu/publisher.ts +++ b/src/client/sfu/publisher.ts @@ -1175,8 +1175,16 @@ export class SfuPublisher { audioTrackIdentifier: identity.audioTrack?.id ?? null, }, ); + const videoEncodings = identity.videoSender.getParameters().encodings; const metrics = { ...connectionMetrics, + videoEncodingCount: + videoEncodings.length > 0 ? videoEncodings.length : null, + activeVideoEncodingCount: + videoEncodings.length > 0 + ? videoEncodings.filter((encoding) => encoding.active !== false) + .length + : null, bitrateKbps: nativeQuality.bitrateKbps ?? connectionMetrics.bitrateKbps, nativeEdgeQualityState: nativeQuality.nativeEdgeQualityState, diff --git a/src/client/types.ts b/src/client/types.ts index 97d0230..b201069 100644 --- a/src/client/types.ts +++ b/src/client/types.ts @@ -19,6 +19,8 @@ export interface ConnectionMetrics { rtpSsrc: number | null; rtpMid: string | null; rtpRid: string | null; + videoEncodingCount: number | null; + activeVideoEncodingCount: number | null; trackIdentifier: string | null; selectedCandidatePairId: string | null; captureWidth: number | null; @@ -103,6 +105,8 @@ export const EMPTY_METRICS: ConnectionMetrics = { rtpSsrc: null, rtpMid: null, rtpRid: null, + videoEncodingCount: null, + activeVideoEncodingCount: null, trackIdentifier: null, selectedCandidatePairId: null, captureWidth: null, diff --git a/src/client/webrtc/video-codec-preflight.ts b/src/client/webrtc/video-codec-preflight.ts index 30f3597..dd7a904 100644 --- a/src/client/webrtc/video-codec-preflight.ts +++ b/src/client/webrtc/video-codec-preflight.ts @@ -285,19 +285,6 @@ function delay(milliseconds: number, signal?: AbortSignal): Promise { }); } -async function waitForIceGathering( - connection: RTCPeerConnection, - deadline: number, - signal?: AbortSignal, -): Promise { - while (connection.iceGatheringState !== "complete") { - if (Date.now() >= deadline) { - throw new Error("H.264 preflight timed out"); - } - await delay(PREFLIGHT_POLL_MS, signal); - } -} - async function runH264Probe( track: MediaStreamTrack, profile: QualityProfile, @@ -305,6 +292,39 @@ async function runH264Probe( ): Promise { const senderConnection = new RTCPeerConnection({ iceServers: [] }); const receiverConnection = new RTCPeerConnection({ iceServers: [] }); + // This in-process probe can start encoding as soon as one local pair works; + // gathering every candidate must not extend the codec decision. + const senderCandidates: RTCIceCandidate[] = []; + const receiverCandidates: RTCIceCandidate[] = []; + let senderRemoteReady = false; + let receiverRemoteReady = false; + let iceFailed = false; + const addCandidate = async ( + target: RTCPeerConnection, + candidate: RTCIceCandidate, + ): Promise => { + try { + await target.addIceCandidate(candidate); + } catch { + iceFailed = true; + } + }; + senderConnection.addEventListener("icecandidate", (event) => { + if (!event.candidate) return; + if (receiverRemoteReady) { + void addCandidate(receiverConnection, event.candidate); + } else { + senderCandidates.push(event.candidate); + } + }); + receiverConnection.addEventListener("icecandidate", (event) => { + if (!event.candidate) return; + if (senderRemoteReady) { + void addCandidate(senderConnection, event.candidate); + } else { + receiverCandidates.push(event.candidate); + } + }); const abort = () => { senderConnection.close(); receiverConnection.close(); @@ -325,22 +345,32 @@ async function runH264Probe( await senderConnection.setLocalDescription( await senderConnection.createOffer(), ); - await waitForIceGathering(senderConnection, deadline, signal); await receiverConnection.setRemoteDescription( senderConnection.localDescription!, ); + receiverRemoteReady = true; + await Promise.all( + senderCandidates.splice(0).map((candidate) => + addCandidate(receiverConnection, candidate), + ), + ); await receiverConnection.setLocalDescription( await receiverConnection.createAnswer(), ); - await waitForIceGathering(receiverConnection, deadline, signal); await senderConnection.setRemoteDescription( receiverConnection.localDescription!, ); + senderRemoteReady = true; + await Promise.all( + receiverCandidates.splice(0).map((candidate) => + addCandidate(senderConnection, candidate), + ), + ); let warmupBaseline: H264ProbeSample | null = null; let measurementBaseline: H264ProbeSample | null = null; while (Date.now() < deadline) { - if (signal?.aborted || track.readyState === "ended") { + if (signal?.aborted || track.readyState === "ended" || iceFailed) { return false; } const sample = readH264ProbeSample( diff --git a/src/server/hybrid-media-router.ts b/src/server/hybrid-media-router.ts index 53ed6ab..90fce68 100644 --- a/src/server/hybrid-media-router.ts +++ b/src/server/hybrid-media-router.ts @@ -477,6 +477,10 @@ export class HybridMediaRouter { message.diagnostics.mediaSourceFramesPerSecond ?? null, width: message.diagnostics.width ?? null, height: message.diagnostics.height ?? null, + videoEncodingCount: + message.diagnostics.videoEncodingCount ?? null, + activeVideoEncodingCount: + message.diagnostics.activeVideoEncodingCount ?? null, availableOutgoingKbps: message.diagnostics.availableOutgoingKbps ?? null, rttMs: message.diagnostics.rttMs ?? null, diff --git a/src/shared/protocol.ts b/src/shared/protocol.ts index d6471a6..327ba14 100644 --- a/src/shared/protocol.ts +++ b/src/shared/protocol.ts @@ -649,6 +649,12 @@ const senderQualityDiagnosticsSchema = z mediaSourceFramesPerSecond: nullableEvidenceNumber(240).optional(), width: nullableEvidenceInteger(16_384).optional(), height: nullableEvidenceInteger(16_384).optional(), + videoEncodingCount: nullableEvidenceInteger( + Number.MAX_SAFE_INTEGER, + ).optional(), + activeVideoEncodingCount: nullableEvidenceInteger( + Number.MAX_SAFE_INTEGER, + ).optional(), availableOutgoingKbps: nullableEvidenceNumber(100_000).optional(), rttMs: nullableEvidenceNumber(60_000).optional(), packetLossPercent: nullableEvidenceNumber(100).optional(), diff --git a/tests/peer-assisted-benchmark.test.ts b/tests/peer-assisted-benchmark.test.ts index 1dacf81..4064ac4 100644 --- a/tests/peer-assisted-benchmark.test.ts +++ b/tests/peer-assisted-benchmark.test.ts @@ -17,11 +17,14 @@ import { joinViewerBurst, mergeRecoveryHostPeaks, parseBenchmarkCanaryMode, + parseBenchmarkCodecMode, parseBenchmarkConfig, parseExpectedEndpointCap, + parsePagePerformanceMetrics, parseViewerCounts, sanitizeFailurePageEvidence, summarizeBenchmarkRouteTiming, + summarizePageRuntimeResources, summarizeSamples, } from "../scripts/peer-assisted-benchmark"; import { MAX_VIEWERS_PER_ROOM_LIMIT } from "../src/shared/protocol"; @@ -442,6 +445,15 @@ describe("peer topology loopback configuration", () => { } }); + it("uses one explicit benchmark codec mode", () => { + expect(parseBenchmarkCodecMode(undefined)).toBe("auto"); + expect(parseBenchmarkCodecMode("VP8")).toBe("vp8"); + expect(parseBenchmarkCodecMode("h264")).toBe("h264"); + expect(() => parseBenchmarkCodecMode("vp9")).toThrow( + /vp8, auto, or h264/, + ); + }); + it("requires a relay-sized case for the optional quality control smoke", () => { expect(() => parseBenchmarkConfig({ @@ -717,7 +729,13 @@ describe("peer topology loopback observations", () => { rtpStatsId: "host-rtp", bitrateKbps: bitrate, framesPerSecond: fps, + captureFramesPerSecond: 30, + mediaSourceFramesPerSecond: 29, resolution: fps === 25 ? "640x360" : "1280x720", + codec: "video/VP8", + encoderImplementation: "test-encoder", + powerEfficientEncoder: true, + scalabilityMode: "L1T2", availableOutgoingKbps: bitrate * 4, intervalFramesEncoded: intervalFrames, intervalEncodeTimeMs: intervalTime, @@ -747,7 +765,13 @@ describe("peer topology loopback observations", () => { unknownIdentitySamples: 0, bitrateKbps: { sampleCount: 3, min: 1_000, max: 1_400, mean: 1_200 }, framesPerSecond: { sampleCount: 3, min: 25, max: 30 }, + captureFramesPerSecond: { sampleCount: 3, min: 30, max: 30 }, + mediaSourceFramesPerSecond: { sampleCount: 3, min: 29, max: 29 }, resolutions: ["1280x720", "640x360"], + codecs: ["video/VP8"], + encoderImplementations: ["test-encoder"], + scalabilityModes: ["L1T2"], + powerEfficientEncoderSamples: { true: 3, false: 0, unknown: 0 }, encodeIntervals: { sampleCount: 2, framesEncoded: 50, encodeTimeMs: 115 }, qualityLimitationReasonSamples: { none: 2, cpu: 1 }, }); @@ -763,6 +787,70 @@ describe("peer topology loopback observations", () => { expect(summary.browserProcessResources.averageCpuUtilizationPercent).toBeCloseTo(50); }); + it("parses and summarizes per-page main-thread resource deltas", () => { + const metrics = ( + timestamp: number, + task: number, + script: number, + layout: number, + recalc: number, + heap: number, + ) => ({ + timestampSeconds: timestamp, + taskDurationSeconds: task, + scriptDurationSeconds: script, + layoutDurationSeconds: layout, + recalcStyleDurationSeconds: recalc, + jsHeapUsedBytes: heap, + }); + expect( + parsePagePerformanceMetrics([ + { name: "TaskDuration", value: 1 }, + { name: "Timestamp", value: 10 }, + { name: "ScriptDuration", value: 0.4 }, + { name: "LayoutDuration", value: 0.1 }, + { name: "RecalcStyleDuration", value: 0.05 }, + { name: "JSHeapUsedSize", value: 10_000 }, + { name: "UnrelatedMetric", value: 99 }, + ]), + ).toEqual(metrics(10, 1, 0.4, 0.1, 0.05, 10_000)); + expect(parsePagePerformanceMetrics([])).toBeNull(); + + const before = [ + { + ...page("host", "host", 2, 0), + performance: metrics(10, 1, 0.4, 0.1, 0.05, 10_000), + }, + { + ...page("viewer", "viewer-1", 0, 1), + performance: metrics(10, 2, 0.8, 0.2, 0.1, 20_000), + }, + { + ...page("viewer", "viewer-2", 1, 1), + performance: metrics(10, 3, 1.2, 0.3, 0.15, 30_000), + }, + { ...page("viewer", "viewer-3", 0, 1), performance: null }, + ]; + const after = structuredClone(before); + after[0]!.performance = metrics(12, 1.4, 0.55, 0.12, 0.07, 12_000); + after[1]!.performance = metrics(12, 2.1, 0.84, 0.21, 0.11, 22_000); + after[2]!.performance = metrics(12, 3.3, 1.3, 0.33, 0.17, 33_000); + + const summary = summarizePageRuntimeResources([ + { atEpochMs: 2_000, elapsedMs: 0, pages: before }, + { atEpochMs: 4_000, elapsedMs: 2_000, pages: after }, + ]); + expect(summary.host.validIntervals).toBe(1); + expect(summary.host.averageTaskUtilizationPercent).toBeCloseTo(20); + expect(summary.host.peakJsHeapUsedBytes).toBe(12_000); + expect(summary.viewer.averageTaskUtilizationPercent).toBeCloseTo(5); + expect(summary.viewer.invalidIntervals).toBe(0); + expect(summary.relay.averageTaskUtilizationPercent).toBeCloseTo(15); + expect(summary.host.scriptTimeSeconds).toBeCloseTo(0.15); + expect(summary.host.layoutTimeSeconds).toBeCloseTo(0.02); + expect(summary.host.recalcStyleTimeSeconds).toBeCloseTo(0.02); + }); + it("checks one endpoint cap for Host and Viewer senders", () => { const initial = [ page("host", "host", 2, 0), diff --git a/tests/sfu-client.test.ts b/tests/sfu-client.test.ts index 4fbc1b8..d33ff42 100644 --- a/tests/sfu-client.test.ts +++ b/tests/sfu-client.test.ts @@ -559,6 +559,10 @@ describe("SfuPublisher", () => { await publisher.activate(stream(video), qualityProfile); const sender = livekit.state.rooms[0].localParticipant.publications[0].track .sender; + sender.parameters.encodings = [ + { rid: "q", active: true }, + { rid: "f", active: true }, + ]; sender.getStats .mockResolvedValueOnce( simulcastSenderReport(video.id, 1_000, 100_000, 200_000), @@ -573,6 +577,8 @@ describe("SfuPublisher", () => { rtpRid: "f", frameWidth: 1920, frameHeight: 1080, + videoEncodingCount: 2, + activeVideoEncodingCount: 2, }); expect( sfuPublisherQualityEvidenceFromMetrics( @@ -593,6 +599,10 @@ describe("SfuPublisher", () => { ).toMatchObject({ state: "healthy", sampleTimestampMs: 3_000, + diagnostics: { + videoEncodingCount: 2, + activeVideoEncodingCount: 2, + }, }); }); @@ -607,6 +617,10 @@ describe("SfuPublisher", () => { await publisher.activate(stream(video), qualityProfile); const sender = livekit.state.rooms[0].localParticipant.publications[0].track .sender; + sender.parameters.encodings = [ + { rid: "q", active: true }, + { rid: "f", active: false }, + ]; sender.getStats.mockResolvedValueOnce( simulcastSenderReport(video.id, 1_000, 100_000, 200_000, false), ); @@ -617,6 +631,8 @@ describe("SfuPublisher", () => { rtpRid: "q", frameWidth: 960, frameHeight: 540, + videoEncodingCount: 2, + activeVideoEncodingCount: 1, }); });