From 0af8e90284b765cedddcc782fe4167fe1a2f4537 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Sat, 22 Aug 2026 04:23:55 +0800 Subject: [PATCH] feat(observability): expose local media path evidence (#166) Add local-only SFU, encoder, A/V sync, and selected candidate evidence without changing media routing or wire contracts. --- docs/project-memory.md | 10 +- docs/research/browser-screen-audio-quality.md | 20 +- docs/research/realtime-quality-adaptation.md | 56 +++-- docs/research/webrtc-p2p-screen-sharing.md | 3 +- docs/status.md | 11 +- docs/方案设计.md | 4 +- src/client/components/StatsGrid.tsx | 114 +++++++++- src/client/media/host-sfu-route.ts | 50 ++++- src/client/pages/HostPage.tsx | 32 ++- src/client/sfu/publisher.ts | 119 ++++++++++ src/client/types.ts | 26 +++ src/client/webrtc/host-peer.ts | 34 +-- src/client/webrtc/stats.ts | 201 ++++++++++++++++- tests/client-state.test.ts | 207 ++++++++++++++++++ tests/route-transition.test.ts | 75 ++++++- tests/sfu-client.test.ts | 181 ++++++++++++++- tests/stats-grid.test.ts | 79 +++++++ tests/viewer-quality-evidence.test.ts | 15 ++ 18 files changed, 1150 insertions(+), 87 deletions(-) diff --git a/docs/project-memory.md b/docs/project-memory.md index d6e702a..5202935 100644 --- a/docs/project-memory.md +++ b/docs/project-memory.md @@ -31,7 +31,7 @@ Last updated: 2026-08-22 - C+B stays edge-local: severe keeps its fallback ladder; relative relay FPS uses fresh parent evidence and peers only. Two confirmed children under one Viewer relay/session/share within 5s pause its eligibility for 30s and peer-only evacuate its children; no score. - Flagship media is UDP; HTTPS/WSS stays TLS/TCP; the old release remains rollback-only. - Treat settings as ceilings and degradation as unclassified. Use correlated Host A+B/Viewer C and one-variable evidence; never force AV1, infer by UA, or create a composite score. -- Screen audio: Web capture requests speech processing off and ideal stereo. Share advanced settings offer 64/128/256 kbps sender ceilings, default 128, and lock the choice for the active share across P2P, browser relay, and SFU. Peer answers permit stereo up to 256 kbps; SFU keeps `forceStereo`, DTX off, and RED retained. Proof remains open; voice and Native stay separate. +- Screen audio source candidate: Web capture requests speech processing off and ideal stereo. Share advanced settings offer 64/128/256 kbps sender ceilings, default 128, and lock the choice for the active share across P2P, browser relay, and SFU. Peer answers permit stereo up to 256 kbps; SFU keeps `forceStereo`, DTX off, and RED retained. Local Viewer details add non-uploaded A/V playout, jitter-buffer and concealment evidence. Browser/route proof remains open; voice and Native stay separate. - Presence reports active upstream for topology; assignment alone never proves media. A Web SFU Viewer becomes connected only after session-bound first media; track/route/session loss clears it. Status/details stay neutral until current P2P/SFU/TURN evidence. Names stay socket/localStorage-only; Native remains outside. - Do not add scene detection, dynamic-FPS control, or forced AV1 without negotiation, encode, game, CPU/GPU, and sender evidence. - ADR-0002 grant/public access, the v3 room-password migration, and Web Host display names are deployed. Names/presence remain session-only without account or roster tables. @@ -43,16 +43,16 @@ Last updated: 2026-08-22 ## Current Implementation - The repository is one npm package using Node.js 24, React, TypeScript, Vite, native WebRTC, `ws`, Zod, Vitest, and separate coturn. -- Production: exact `083c111437485064a766368944a7938b79e07365`, release `083c11143748`, Web wire `screener-v5`; `27ad90ddf9f8` remains the rollback release. +- Production: exact `083c111437485064a766368944a7938b79e07365`, release `083c11143748`, Web wire `screener-v5`; `27ad90ddf9f8` is the rollback release. - Web keeps one SFU stream, clears unavailable video, exposes evidence-backed routes/transports and paused preview, and drops signals to a grace-retained offline current-edge target. Routing is automatic; entry/nickname UI is live; Native remains source-only. -- Web relay is cap2 with one upstream; child3 rejects and no UA/visibility split exists. -- Deployed relay evidence reaches opted-in Host rosters; direct proof stays local-edge-only. +- Production Web endpoints use cap2 with one upstream; source supports an explicit deployment cap from one through three without a UA/visibility split. +- Relay evidence is opt-in; direct proof and selected-pair addresses stay local, never in relayed evidence. - `screener`, LiveKit, coturn, and nginx are active; `screener` has `NRestarts=0`, local/public health are 200, and the endpoint-cap environment is unset so the parameterized default is two. SQLite v3 has five rooms, SHA-256 `aef724ae52c4107be8ec8791e72f8dcaa11b0ec849fb432d022e2cfb9cfe0974`, owner/mode `screener:screener`/0600. - Production keeps ordinary ICE STUN-only, SFU roots <=2, selected-edge UDP TTL 120, and one pending/answered `peer-selected` attempt per room excluding Host ingress. This rollout ran no browser/media/SFU/TURN/performance canary; earlier exact-`16f6eab` evidence proves active Host ingress only. - Production requires independent `SITE_ACCESS_PASSWORD`: its stateless cookie authorizes creation/Host and code-only Viewer attempts; private grants stay direct, while public/private code entry needs site access alone/plus room password. Env/endpoint/cookie/nginx naming migrated atomically; failures remain neutral; there are no accounts/JWT/session rows. Local `gate:access-privacy` proves fragment isolation, site/grant transport-log absence, and raw room-password SQLite absence; room-password WebSocket/log ingress and live/headful evidence remain open. - Production uses nginx, Node.js 24.19.0, coturn 4.17.2, and LiveKit 1.13.5 on UDP 7882 (TCP fallback off). Ordinary peer ICE receives no TURN; the selected-edge tuple is configured only for the current controller edge, with no credential pre-advertised to ordinary peers. Coturn retains the old authenticated-relay ports. Services are healthy with zero restarts. - ADR-0005 rejects `PEER_ASSISTED_ROOM_IDS`; `PEER_ASSISTED_MEDIA=true` enables every room and ordinary peers stay STUN-only. Initial or active Host ingress may consume one bound relay-only grant; ready/abort clears it and failure restores peer baseline. Stale room-ID config fails startup. Production and source-only Native use `screener-v5`; Native does not subscribe to Web presence. -- Defaults are balanced with automatic codec and 128 kbps screen audio; clarity/fluid, H.264/VP8, and 64/128/256 kbps audio remain explicit. Codec and audio quality are locked during a share. Web SFU uses LiveKit 2.22.0 `q,h`; SFU/UDP and selected-TURN work; BWE and Host A+B remain open. +- Defaults are balanced with automatic codec and 128 kbps screen audio; clarity/fluid, H.264/VP8, and 64/128/256 kbps audio remain explicit. Codec and audio quality are locked during a share. LiveKit 2.22.0 SFU uses `q,h`; this source branch adds one Host-local current-`h` A+B snapshot and clears it on publisher identity changes. BWE and production observation remain open. - Chrome 151 five-Viewer 720p30 cap2 evidence: Host/relay fanout two; both relay children +80 decoded frames/~1.1 MB; every Viewer decoded; clean stop. Functional only; resource and heterogeneous-network behavior remain open. ## Provisional Quality Targets diff --git a/docs/research/browser-screen-audio-quality.md b/docs/research/browser-screen-audio-quality.md index 44b145f..b12aae5 100644 --- a/docs/research/browser-screen-audio-quality.md +++ b/docs/research/browser-screen-audio-quality.md @@ -1,6 +1,6 @@ # Browser Screen-Audio Quality Controls -Accessed: 2026-08-21 +Accessed: 2026-08-22 Status: peer and SFU routes already use stereo and a 128 kbps default, but users still report speech-gated movie/game audio, including on a phone connected @@ -277,9 +277,18 @@ Audio diagnostics keep four layers separate: not uploaded or persisted. 4. **Observed:** current local details derive interval audio bitrate from byte deltas and interval loss from `lostDelta / (receivedDelta + lostDelta)`, and - display reported jitter. The first sample, changed identity, reset, negative - delta, zero denominator or absent field is unknown. Concealment, jitter - buffer and FEC counters remain an acceptance-matrix follow-up. + display reported jitter. For the unique current inbound screen video/audio + pair, they also display audio-minus-video `estimatedPlayoutTimestamp`, + interval average jitter-buffer delay as + `delta(jitterBufferDelay) / delta(jitterBufferEmittedCount)`, audio concealed + samples as `delta(concealedSamples) / delta(totalSamplesReceived)`, and the + interval `concealmentEvents` count. RTP stats id, SSRC or track-identifier + changes rebase that media kind; a counter reset rebases the affected + interval. The first sample, negative delta, zero denominator, ambiguous RTP + object or absent field is unknown. These local-only fields are not added to + Viewer quality-evidence signaling or persistence. Browser availability and + actual route-switch synchronization remain target-device evidence; FEC + counters remain an acceptance-matrix follow-up. Never substitute `48000`, `2`, `false`, or `0` for an unavailable field. @@ -342,6 +351,9 @@ Preserve the existing persistent Viewer `MediaStream`/media element and add or remove its audio/video tracks in place. Introducing a separate audio element or Web Audio graph would create a new clock/buffering boundary and is outside this fix; the stable stream already deployed is the A/V synchronization baseline. +The local receiver diagnostic observes only `getStats()` on that existing +route. It does not set `jitterBufferTarget`, create a clock, modify playout, add +a wire field, or claim that a missing browser statistic is zero. The implementation budget is deliberately bounded to dependency manifest/lockfile entries, one SDP helper, `ViewerPeer` answer wiring, SFU diff --git a/docs/research/realtime-quality-adaptation.md b/docs/research/realtime-quality-adaptation.md index 0a1d342..71a0a0e 100644 --- a/docs/research/realtime-quality-adaptation.md +++ b/docs/research/realtime-quality-adaptation.md @@ -1,6 +1,6 @@ # Realtime Screen-Share Quality Adaptation -- Research date: 2026-08-21 +- Research date: 2026-08-22 - Scope: realtime game screen sharing in the browser - Status: implementation input; real-device quality remains unverified @@ -209,11 +209,13 @@ does not prove shared encode: separate browser PeerConnections may construct separate encoders, and the SFU still publishes only the configured `q,h` representations. -The retained follow-up is one direct-Viewer comparison using existing stats, -not a new benchmark: actual codec/profile, encoder implementation, power -efficiency, source/send/receive FPS, encode time and limitation reason. It must -distinguish software fallback from a hardware queue or driver limit before any -H.264-specific policy changes. +The retained follow-up is one target-device H.264/VP8 comparison using these +local diagnostics, not a new benchmark framework: actual codec/profile, +encoder implementation, power efficiency, configured/source/send/receive FPS, +interval encoded frames/encode time, and limitation reason. Missing +implementation or power-efficiency fields remain unknown. No H.264 root cause +or policy change is accepted until the controlled sample distinguishes capture +starvation, software fallback, hardware queue/driver pressure, and congestion. Open-source distribution is not itself a patent-license exemption. This Web change only requests a codec already implemented by the browser/LiveKit path and @@ -257,14 +259,16 @@ privacy boundary are owned by Verified specification facts: the W3C stats model supports the needed separation but does not produce the product decision itself. -`MediaStreamTrack.getSettings()` supplies the actual -capture dimensions/frame rate. Outbound RTP exposes emitted dimensions/FPS, -byte and retransmission counters, encode time, target bitrate when available, -and the current `qualityLimitationReason`; the selected candidate pair can -expose RTT and available outgoing bitrate. Inbound RTP exposes received -dimensions/FPS, byte/loss/jitter counters, decoded/dropped frames, and freeze -counters when implemented. Stats members may be absent, and cumulative values -must be compared across two samples rather than treated as interval values. +`MediaStreamTrack.getSettings()` supplies the track's current configured +dimensions/frame rate, not the recent frame cadence. `RTCVideoSourceStats` +supplies the last-second FPS actually fed to the encoder. Outbound RTP exposes +emitted dimensions/FPS, byte and retransmission counters, encode time, target +bitrate when available, and the current `qualityLimitationReason`; the selected +candidate pair can expose RTT and available outgoing bitrate. Inbound RTP +exposes received dimensions/FPS, byte/loss/jitter counters, decoded/dropped +frames, and freeze counters when implemented. Stats members may be absent, and +cumulative values must be compared across two samples rather than treated as +interval values. The user-facing loss value should therefore be an RTP interval loss rate, not the cumulative `packetsLost` counter and not a claim about UDP itself. For one @@ -281,8 +285,8 @@ Correlate one time interval and media generation across: | Evidence | Fields | | --- | --- | -| A. Host capture | actual width, height, FPS from `getSettings()` | -| B. Host outbound | width/FPS/bitrate, target/available bitrate, interval encode time, limitation reason, path, RTT, loss/retransmission, and derived negotiated codec/profile/parameters plus applicable `scalabilityMode` | +| A. Host capture | configured width, height, FPS from `getSettings()` plus observed `media-source` input FPS | +| B. Host outbound | RID, width/FPS/bitrate, target/available bitrate, interval encoded frames/encode time, limitation reason, path, RTT, loss/retransmission, and derived negotiated codec/profile/parameters plus applicable `scalabilityMode` | | C. Viewer inbound | width/FPS/bitrate, loss, jitter, interval decode/drop/freeze, corresponding derived codec/profile/parameters/layer, and actual decode behavior | Product inference from those facts: use the following ordered classification: @@ -306,11 +310,19 @@ device/network identifiers. Server-authoritative path and connection generations provide authorization and correlation; a general remote stats stream or telemetry pipeline is unnecessary. -The repository implements the local host A+B foundation: same-tick capture -settings plus one uniquely matched outbound RTP sample, explicit sample/media -identity and adjacent deltas, `remoteId` linkage, and the selected path reached -through that RTP stream's transport. Source replacement blocks sampling and -invalidates in-flight generations. +The repository implements the local host A+B foundation: same-tick track +settings, linked `media-source` input FPS, and one uniquely matched outbound RTP +sample with its RID, explicit sample/media identity and adjacent frame/encode +time deltas, `remoteId` linkage, and the selected path reached through that RTP +stream's transport. Source replacement blocks sampling and invalidates in-flight +generations. Direct/relay edges bind that evidence to the +current track and PeerConnection. The Host SFU publisher also samples only its +active LiveKit sender's `h` representation and binds the result again to the +current publication generation; replacement, profile reset, retirement, +disconnect, and authoritative resync clear the old identity before another +sample can appear. The Host details UI renders that source once as `SFU 发送`, +not once per SFU-fed Viewer. This is local diagnostics only: it creates no wire, +score, selector, or media action. Authenticated Viewer C is also implemented for each current ordinary or peer-assisted P2P hop. A viewer sends one nullable, sanitized aggregate window @@ -704,6 +716,7 @@ is a separate optimization. - [W3C MediaStreamTrack Content Hints](https://www.w3.org/TR/mst-content-hint/) - [W3C Screen Capture](https://www.w3.org/TR/screen-capture/) +- [W3C Media Capture and Streams `getSettings()`](https://www.w3.org/TR/mediacapture-streams/#dom-mediastreamtrack-getsettings) - [W3C WebRTC](https://www.w3.org/TR/webrtc/) - [W3C WebRTC codec preferences](https://www.w3.org/TR/webrtc/#dom-rtcrtptransceiver-setcodecpreferences) - [RFC 7742 WebRTC video codec requirements](https://www.rfc-editor.org/rfc/rfc7742.html) @@ -744,6 +757,7 @@ is a separate optimization. - [LiveKit server 1.13.5 per-subscriber layer application](https://github.com/livekit/livekit/blob/v1.13.5/pkg/rtc/subscribedtrack.go) - [LiveKit server 1.13.5 codec-specific layer selectors](https://github.com/livekit/livekit/blob/v1.13.5/pkg/sfu/forwarder.go) - [LiveKit client 2.22.0 Dynacast and saved degradation preference](https://github.com/livekit/client-sdk-js/blob/v2.22.0/src/room/track/LocalVideoTrack.ts) +- [LiveKit client 2.22.0 local sender stats](https://github.com/livekit/client-sdk-js/blob/v2.22.0/src/room/track/LocalTrack.ts) - [LiveKit server 1.13.5 Dynacast quality aggregation](https://github.com/livekit/livekit/blob/v1.13.5/pkg/rtc/dynacast/dynacastqualityvideo.go) - [LiveKit server 1.13.5 enabled-quality generation](https://github.com/livekit/livekit/blob/v1.13.5/pkg/rtc/dynacast/dynacastmanagervideo.go) - [LiveKit server 1.13.5 release assets and checksums](https://github.com/livekit/livekit/releases/tag/v1.13.5) diff --git a/docs/research/webrtc-p2p-screen-sharing.md b/docs/research/webrtc-p2p-screen-sharing.md index b5f7c34..37cac3c 100644 --- a/docs/research/webrtc-p2p-screen-sharing.md +++ b/docs/research/webrtc-p2p-screen-sharing.md @@ -2,6 +2,7 @@ - 调研日期：2026-08-18 - 移动端采集与 Viewer 投屏能力复核：2026-08-21 +- selected candidate 地址隐私语义复核：2026-08-22 - 目标场景：一名玩家向少量熟人私密分享，观看者可用手机/桌面浏览器加入，低延迟，尽量不消耗媒体服务器带宽 - 结论状态：本文记录已部署 PoC 的 P2P/coturn 基线。ADR-0005 与[低服务器成本媒体路由](./low-server-media-routes.md)已取代本文早期“每条 peer edge 必带 TURN”的旗舰建议；生产后续移除了 room `1` 边界，ordinary ICE 仍为 STUN-only，selected-edge TURN 已配置但尚未完成真实媒体验收 @@ -318,7 +319,7 @@ WebRTC 媒体本身使用 DTLS-SRTP 加密，但 direct P2P 仍可能让这组 - `qualityLimitationReason` 和各原因累计时长 - jitter buffer delay、decode time 和 total packet send delay -`RTCIceCandidateStats.protocol` 是内部候选传输字段；只有本地 relay candidate 的 `relayProtocol` 才能确认本端到 TURN 的实际传输。规范不向远端暴露 `relayProtocol`，所以不能从 `remoteCandidate.protocol` 推断远端 TURN 传输。当前详情以 `P2P`/`SFU fallback` 表示媒体方式，直连只显示实际 `protocol`，relay 先显示 `TURN`，仅本地 relay 再附加 `/UDP` 等实际值；candidate type 单列为候选路径，不为补齐远端字段扩展信令或遥测。 +`RTCIceCandidateStats.protocol` 是内部候选传输字段；只有本地 relay candidate 的 `relayProtocol` 才能确认本端到 TURN 的实际传输。规范不向远端暴露 `relayProtocol`，所以不能从 `remoteCandidate.protocol` 推断远端 TURN 传输。当前详情以 `P2P`/`SFU fallback` 表示媒体方式，直连只显示实际 `protocol`，relay 先显示 `TURN`，仅本地 relay 再附加 `/UDP` 等实际值；candidate type 单列为候选路径，不为补齐远端字段扩展信令或遥测。W3C Stats 规定远端 candidate `address` 默认可为 `null`，浏览器也可按隐私策略过滤；诊断 UI 因此只读取当前本地 report 中由 media transport、`selectedCandidatePairId`、`localCandidateId`/`remoteCandidateId` 精确关联的地址与端口，缺失就保持未知，不解析 SDP 补值，也不上传、记录或持久化。 WebRTC 标准没有承诺固定毫秒延迟。工程目标必须带网络条件，并使用画面时间码或高速摄像机测量玻璃到玻璃延迟。60 fps 的单帧周期是 16.7 ms，端到端延迟还包含采集等待、编码、单程网络、jitter buffer、解码和显示。 diff --git a/docs/status.md b/docs/status.md index 0f5f8ff..e2b0639 100644 --- a/docs/status.md +++ b/docs/status.md @@ -4,7 +4,7 @@ Last updated: 2026-08-22 ## Phase -Production is exact `083c111437485064a766368944a7938b79e07365` at `https://share.bonfire.icu`, release `083c11143748`, wire `screener-v5`. SQLite v3/five rooms keeps SHA-256 `aef724ae52c4107be8ec8791e72f8dcaa11b0ec849fb432d022e2cfb9cfe0974`, owner/mode `screener:screener`/0600. `screener`, LiveKit, coturn, and nginx are active; local/public health are green and `screener` has `NRestarts=0`. The previous `27ad90ddf9f8` release remains the rollback target. +Production is exact `083c111437485064a766368944a7938b79e07365` at `https://share.bonfire.icu`, release `083c11143748`, wire `screener-v5`. SQLite v3/five rooms keeps SHA-256 `aef724ae52c4107be8ec8791e72f8dcaa11b0ec849fb432d022e2cfb9cfe0974`, owner/mode `screener:screener`/0600. `screener`, LiveKit, coturn, and nginx are active; local/public health are green and `screener` has `NRestarts=0`. The previous `27ad90ddf9f8` release is the rollback target. Ordinary ICE is STUN-only; limits are SFU roots <=2, one `peer-selected` attempt excluding Host ingress, and selected UDP TTL 120. No browser/media/SFU/TURN/performance canary ran; exact-`16f6eab` Host-ingress proof is separate. Stable SFU, route/transport truth, preview notice, entry/nickname UI, selected-edge lease authority, and corroborated bad-relay demotion are live. `MAX_PEER_RELAY_DOWNSTREAM_EDGES` is unset, so Web endpoints use the parameterized default of two. Native is source-only. @@ -14,11 +14,12 @@ Parallel flagship work; use minimum relevant checks and rollback proof. Target-d ## Current Snapshot -- Capture precedes rooms; source/quality changes and AV pause preserve peers. Defaults are balanced, automatic codec, and 128 kbps screen audio. Clarity/fluid plus H.264/VP8 stay explicit. Source adds 64/128/256 kbps audio presets under Share advanced settings; codec and audio quality are locked during a share. Browser degradation, audible route proof, and Host SFU A+B remain open. +- Capture precedes rooms; source/quality changes and AV pause preserve peers. Defaults are balanced, automatic codec, and 128 kbps screen audio. Clarity/fluid plus H.264/VP8 stay explicit. The source candidate adds 64/128/256 kbps audio presets under Share advanced settings; codec and audio quality are locked during a share. Browser degradation, audible route proof, and Host SFU A+B remain open. - Deployed `screener-v5` admits private fragment grants directly; code-only Viewers need site access, and private rooms additionally need their password. Credentials remain unstored. Web presence fields remain opt-in and Native does not subscribe to them. - Deployed UI has nickname editing, room entry, Viewer roster, duplicate-only ID suffixes, pause notice, and session-bound SFU first-media truth that clears on track/route/session loss. P2P/SFU/TURN labels require evidence. Guarded relay evidence reaches opted-in Web Host rosters without Host-local transport claims or false parent-edge proof. - Native v5 is source-only (memory rooms, 300s reclaim); it does not subscribe to Web presence, accepts the shared absolute three-child bound, and fails unsupported SFU/selected ingress boundedly. Production uses the default endpoint cap of two. - Production roots/Web relay <=2; Viewer upstream1. Source parameterizes endpoint cap at default2/range1-3 while SFU roots remain <=2; production stays2. One-root healthy-SFU MBB is deployed/media-unverified; source defers one cooldown ready for expiry reproof. Capacity `0 -> 1`, multi-root and browser gates remain open. Room `1` is historical; HTTPS/WSS is TLS/TCP. +- Source-only connection details read selected-pair addresses from the current page's local stats, hide them when incomplete, and never add them to relayed evidence, signaling, server state, logs, or persistence. - After an answer, a generation-bound 15s initial-connect deadline enters ICE restart; success/replacement/disposal cancels it. It is deployed but not mobile-verified. - C+B is uncalibrated. Production compares relay-child FPS with fresh parent input; two corroborated children can pause one Viewer relay for 30s and trigger peer-only reassignment. No candidate keeps the old edge, and admission rescue remains deterministic. - Deployed `screener-v5` selected-TURN authority separates immutable grant revision from current route authority. Unrelated revisions may carry only the same in-budget edge, while topology/session/share/advertised-budget loss revokes it before active authority. Browser/media proof remains open. @@ -27,7 +28,7 @@ Parallel flagship work; use minimum relevant checks and rollback proof. Target-d - Chrome 151 synthetic `1/3/5/8` and 720p30 runs kept bounded fanout/decoding. A five-Viewer cap2 run had one relay serve two children (+80 decoded frames/~1.1 MB each), all decoded and no fatal. Functional only. - Chrome 151 + local LiveKit 1.13.5/client 2.22.0 SFU/UDP passed after `q,f` -> `q,h`: one root, decoded 3 -> 23, rendered 26, Host edge one, clean leaves, no TURN. Functional only. -- Host A+B and authenticated P2P Viewer C are sanitized, generation-bound, read-only, and fail closed for stale, ambiguous, or SFU-fed evidence; no raw media metadata is retained. +- Host A+B/P2P Viewer C are sanitized, generation-bound/read-only. Source adds Host-local current SFU `h` snapshot, cleared on identity change; SFU C stays fail-closed/unverified. - The `a11a73d` built-in TURN canary is rejected: local allocation passed, but direct Host/Pion Viewer failed before forced relay. Full rollback restored STUN-only client ICE and zero allocations. - Chrome 151 loopback proves H.264 negotiation/decode interoperability, not performance. Production orders H.264 first and keeps browser repair/fallback codecs. - Native WGC/MF used no browser capture/encoder. One Chrome Viewer rendered 203 1280x720 frames and received 500 Opus packets; process/LUID-correlated `VideoEncode` was nonzero, with no fatal/encoder errors. @@ -40,7 +41,7 @@ Parallel flagship work; use minimum relevant checks and rollback proof. Target-d ## Unverified Boundaries -- The `6ccb516a` cutover ran no browser/media/SFU/TURN/performance canary. Earlier `16f6eab` proof covers Host ingress only; healthy reselection, audible quality, last mile, external cohorts, resources/performance and game blur remain open. +- The `083c11143748` cutover has service/health proof but no browser/media/SFU/TURN/performance canary. Earlier `16f6eab` proof covers Host ingress only; healthy reselection, audible quality, last mile, external cohorts, resources/performance and game blur remain open. - Mobile is Viewer-only. Its persistent audible media element keeps media/signaling active while the page lives; iOS lock-screen, reclamation and background reconnection remain device gates. AirPlay/system mirroring is local output; live WebRTC `srcObject` has no portable in-app TV-output contract. - Silent partitions can wait 30 to 60 seconds for heartbeat detection before the default 5-second grace; this remains unverified. - Production startup blur remains open. One local Chrome synthetic same-`balanced` A/B kept route/PC/SSRC/track and coincided with 720p -> 1080p without loss/freeze, but natural ramp prevents causal, production, or SFU proof. @@ -57,7 +58,7 @@ Recovery: ICE restart -> same parent -> alternate peer -> SFU. Source-only manua Native WGC/MF passed one-Viewer hardware; package download, Viewer2/FIFO and game A/V remain. Use representative target-device or production evidence for performance, not an ordinary-PC synthetic gate. Ordinary peers stay STUN-only; initial ingress, `peer-selected`, and healthy-reselection browser gates remain open. ADR-0004 still needs its resource/quality matrix; the 20-viewer gate precedes a default change. Mobile uses the same capacity and is a compatibility observation. -Screen audio defaults to 128 kbps; the source-complete 64/128/256 kbps presets propagate through P2P, browser relay, and SFU and lock for the active share. Verify Host settings, negotiated/observed bitrate, audible quality, and route switches. +Screen audio defaults to 128 kbps; the source-complete 64/128/256 kbps presets propagate through P2P, browser relay, and SFU and lock for the active share. A/V/jitter-buffer/concealment details are local-only in this observability branch. Verify Host settings, negotiated/observed bitrate, audible quality, synchronization evidence, and route switches. ## Blockers And Decisions diff --git a/docs/方案设计.md b/docs/方案设计.md index e357f05..b013d60 100644 --- a/docs/方案设计.md +++ b/docs/方案设计.md @@ -178,6 +178,8 @@ Web peer 的最小 stereo 契约只改 Viewer 生成的 answer。实现把 MIT ` SFU publisher 将 64 kbps 映射到 `AudioPresets.musicStereo`、128 kbps 映射到 `musicHighQualityStereo`，256 kbps 使用显式 `maxBitrate`；三者都保留 `forceStereo: true` 与 `dtx: false`。LiveKit 2.22.0 先合并 room publish defaults，其中 `red: true`；本实现不传 `red` 覆盖值，保留既有 RED request。RED 与 Opus in-band FEC 是不同修复机制，保留前者不声称实际启用、替代或等价于后者。只改 SFU 会导致 route switch 行为分叉，因此 P2P/browser relay 与 SFU 必须同一 slice 落地。Viewer 继续在同一个持久 `MediaStream`/媒体元素中增删 screen video/audio track；不新建 audio element、Web Audio mix/resample graph、第二 audio representation 或独立音频时钟。默认每条 outbound edge 的 audio RTP payload ceiling 为 128 kbps，显式最高档为 256 kbps；cap2 节点对应最多 256/512 kbps（不含 RTP/SRTP/UDP/IP overhead）。stereo 仍可能增加编码与网络成本，且浏览器 relay 不保证跨 PeerConnection shared encode。未来 microphone voice 若进入范围，另建独立 source/track 与 AEC、降噪、gain、DTX 设计，不能作用于电影/游戏屏幕音频。详细证据与 Oopz 未知边界见 `docs/research/browser-screen-audio-quality.md`。 +Viewer 本地连接详情只从当前 route 的唯一 inbound screen video/audio stats 读取同步证据：`audio estimatedPlayoutTimestamp - video estimatedPlayoutTimestamp` 表示当前发送端时间线差；音视频 jitter-buffer delay 取相邻窗口累计 delay/count 的比值；音频 concealment 取相邻窗口 concealed/total sample 与 event 增量。每个媒体种类用 stats id、SSRC 和 track identifier 共同守住区间身份，换流或累计计数回退时先重新基线，字段缺失或分母为零时保持未知。该证据不进入 signaling、质量上报或持久化，也不设置 `jitterBufferTarget`、Web Audio 或任何媒体控制参数；真实同步仍由同一持久 `MediaStream`/媒体元素和 WebRTC playout 负责。 + `frameRate` 约束和 sender 的 `maxFramerate` 都是目标或上限；浏览器仍控制实际捕获、重复帧处理、编码和拥塞降级。静态画面可能由浏览器减少实际编码数据，但标准没有保证每个实现以同一种方式节省采集或 GPU 开销。本 PoC 不添加画面差分、定时改约束或自定义动态 FPS 状态机，先用 `framesEncoded`、实际发送码率、`totalEncodeTime`、质量受限原因和主机资源占用测量真实问题。 质量自适应采用 ADR-0007 的 built-in-first 路径隔离双表示，而不是全房间按最差链路降档。Direct/peer 保留每条 PeerConnection 的 stock WebRTC GCC。每条 SFU path 从 `HIGH` ceiling 开始；优先发布共享 `HIGH+LOW` 并让 LiveKit BWE 独立降层和恢复。若该 gate 通过，Screener 不实现媒体层 selector。应用证据只决定下述 parent/leaf 拓扑资格与诊断。`LOW` 可以常驻；关闭空闲层只是资源优化。active representation/layer 数量硬限制为二，不能按 viewer 增长。标准浏览器不保证跨 PeerConnection 共用一个物理编码器，inactive 层也不保证释放 GPU；实际层数、实例数、CPU/GPU、游戏 FPS/p1 low、区间 encode cost、上传与释放行为必须实测。若没有合格 hardware/power-efficient 路径或额外表示破坏游戏/上传预算，则 fail closed，保留健康路径并明确显示弱路径降级或失败。 @@ -318,7 +320,7 @@ hop 按[低服务器成本媒体路由](./research/low-server-media-routes.md) - `/r/{code}` 观看者：private room 首次解析合法 fragment grant 后立即清理地址栏并直达；仅有 code 时先通过 site access，再按 public/private policy 进入。鉴权后主区域最大化远端视频，标题旁显示可复制的紧凑房间号，顶部保留连接状态，并提供本地音量、静音、全屏和重试控制；Host 主动暂停时复用视频舞台遮罩显示“分享者已暂停”，恢复后立即移除。 - 移动端：视频保持稳定宽高比，控制与状态移到视频下方，任何文字和按钮不得溢出或遮挡。 -两端默认显示紧凑在线 roster；Viewer 端复用同一权威 participant snapshot，只列出当前 Viewer（含本机）并在新 snapshot 到达时整体替换，停止分享仅移除已离线 Host，鉴权失效、房间关闭或页面卸载则清空。Host 与 Viewer 的本机显示名在界面中统一标为“昵称”，默认只读，铅笔按钮紧贴昵称并只切换当前控件的输入、保存和取消，唯一名称仅显示名称，重名时才追加 room-scoped `peerId` 短后缀。一个不持久化的“显示连接详情”复选框按页展开媒体方式、传输协议、候选路径、浏览器实际暴露的本地/远端候选地址、codec、RTT、RTP 丢包率、编解码耗时及逐边统计；真实连接失败、音频缺失、持续画质受限和需要用户动作的告警始终可见。显示名/roster 不进入 access credential 或 SQLite。 +两端默认显示紧凑在线 roster；Viewer 端复用同一权威 participant snapshot，只列出当前 Viewer（含本机）并在新 snapshot 到达时整体替换，停止分享仅移除已离线 Host，鉴权失效、房间关闭或页面卸载则清空。Host 与 Viewer 的本机显示名在界面中统一标为“昵称”，默认只读，铅笔按钮紧贴昵称并只切换当前控件的输入、保存和取消，唯一名称仅显示名称，重名时才追加 room-scoped `peerId` 短后缀。一个不持久化的“显示连接详情”复选框按页展开媒体方式、传输协议、候选路径、浏览器实际暴露的本地/远端候选地址、codec、RTT、RTP 丢包率、编解码耗时及逐边统计；地址与端口只从当前页面 `RTCPeerConnection.getStats()` 精确关联的 selected pair 读取，浏览器隐藏或缺失任一字段就不显示，且不进入质量证据、信令、服务端、日志或持久化。真实连接失败、音频缺失、持续画质受限和需要用户动作的告警始终可见。显示名/roster 不进入 access credential 或 SQLite。 用户可见的非直播状态只使用“已停止分享”（分享端已结束当前发布）、“等待开始分享”（观看端等待发布）、“正在连接”与“正在恢复连接”。`room-closed` 显示“房间已关闭”或“房间已过期”，不得误写成“分享已结束”。房间寿命、媒体代次、换父和恢复等内部状态不得作为额外产品文案暴露。 diff --git a/src/client/components/StatsGrid.tsx b/src/client/components/StatsGrid.tsx index 2bb4402..cd1f174 100644 --- a/src/client/components/StatsGrid.tsx +++ b/src/client/components/StatsGrid.tsx @@ -38,6 +38,21 @@ function preferenceLabel(value: number | string): string { return labels[String(value)] ?? String(value); } +function candidateEndpoint( + address: string | null, + port: number | null, +): string | null { + if (address === null || port === null) { + return null; + } + const host = + address.includes(":") && + !(address.startsWith("[") && address.endsWith("]")) + ? `[${address}]` + : address; + return `${host}:${port}`; +} + function Metric({ label, value, title }: { label: string; value: string; title?: string }) { return (
@@ -69,6 +84,21 @@ export function StatsGrid({ : "" }` : "未知"; + const hasPlaybackEvidence = [ + metrics.audioVideoPlayoutDeltaMs, + metrics.videoJitterBufferDelayMs, + metrics.audioJitterBufferDelayMs, + metrics.audioConcealedSamplesPercent, + metrics.intervalAudioConcealmentEvents, + ].some((value) => value !== null); + const localCandidateEndpoint = candidateEndpoint( + metrics.localCandidateAddress, + metrics.localCandidatePort, + ); + const remoteCandidateEndpoint = candidateEndpoint( + metrics.remoteCandidateAddress, + metrics.remoteCandidatePort, + ); const primaryMetrics = ( <> @@ -76,8 +106,14 @@ export function StatsGrid({ label={direction === "send" ? "发送码率" : "接收码率"} value={`${readableNumber(metrics.bitrateKbps)} kbps`} /> - - + + + {localCandidateEndpoint && ( + + )} + {remoteCandidateEndpoint && ( + + )} {metrics.codecProfile && ( @@ -133,6 +183,35 @@ export function StatsGrid({ label="音频抖动" value={`${readableNumber(metrics.audioJitterMs, 1)} ms`} /> + {direction === "receive" && hasPlaybackEvidence && ( + <> + + + + + + + )} {metrics.audioCodecClockRate !== null && ( + + + {metrics.rtpRid && } {metrics.scalabilityMode && ( )} + MediaStream | null; getProfile: () => QualityProfile; reconcileChildren: (childPeerIds: string[]) => void; send: (message: ClientMessage) => boolean; - createPublisher?: (onDisconnected: () => void) => HostPublisherTransport; + onPublisherUpdate?: (snapshot: HostSfuPublisherSnapshot | null) => void; + createPublisher?: ( + onDisconnected: () => void, + onStats: (metrics: ConnectionMetrics | null) => void, + ) => HostPublisherTransport; } export class HostSfuRoute { @@ -138,6 +148,7 @@ export class HostSfuRoute { const active = this.active; this.pending = null; this.active = null; + this.events.onPublisherUpdate?.(null); if (pending) { pending.failed = true; } @@ -208,8 +219,14 @@ export class HostSfuRoute { this.clearPending(); let slot: HostPublisherSlot; const publisher = - this.events.createPublisher?.(() => this.handleFailure(slot)) ?? - new SfuPublisher({ onDisconnected: () => this.handleFailure(slot) }); + this.events.createPublisher?.( + () => this.handleFailure(slot), + (metrics) => this.handlePublisherStats(slot, metrics), + ) ?? + new SfuPublisher({ + onDisconnected: () => this.handleFailure(slot), + onStats: (metrics) => this.handlePublisherStats(slot, metrics), + }); const selectedEdgeTurn = this.selectedEdgeTurn?.revision === message.revision && this.selectedEdgeTurn.publicationGeneration === publicationGeneration && @@ -329,6 +346,7 @@ export class HostSfuRoute { if (!slot) { return; } + this.events.onPublisherUpdate?.(null); this.active = null; await disconnectPublisher(slot.publisher); this.handleFailure(slot); @@ -351,6 +369,7 @@ export class HostSfuRoute { const active = this.active; this.pending = null; this.active = null; + this.events.onPublisherUpdate?.(null); if (active?.active) { await active.publisher.deactivate().catch(() => false); } @@ -548,6 +567,7 @@ export class HostSfuRoute { return; } const active = this.active; + this.events.onPublisherUpdate?.(null); this.active = null; if (active.active) { await active.publisher.deactivate().catch(() => false); @@ -566,6 +586,7 @@ export class HostSfuRoute { this.pending = null; } if (wasActive) { + this.events.onPublisherUpdate?.(null); this.active = null; } @@ -630,6 +651,29 @@ export class HostSfuRoute { private ready(revision: number, phase: MediaRoutePhase): void { this.events.send({ type: "route-ready", revision, phase }); } + + private handlePublisherStats( + slot: HostPublisherSlot, + metrics: ConnectionMetrics | null, + ): void { + if ( + metrics === null || + this.closed || + slot.failed || + !slot.active || + this.active !== slot || + !this.currentActiveToken(slot) + ) { + if (metrics === null && this.active === slot) { + this.events.onPublisherUpdate?.(null); + } + return; + } + this.events.onPublisherUpdate?.({ + metrics: { ...metrics }, + senderParameters: slot.publisher.getSenderParameters?.() ?? null, + }); + } } async function disconnectPublisher( diff --git a/src/client/pages/HostPage.tsx b/src/client/pages/HostPage.tsx index 692a9b9..98bd6c9 100644 --- a/src/client/pages/HostPage.tsx +++ b/src/client/pages/HostPage.tsx @@ -86,7 +86,10 @@ import { type VideoCodecPreference, VIDEO_CODEC_PREFERENCE_LABELS, } from "../media/quality"; -import { HostSfuRoute } from "../media/host-sfu-route"; +import { + HostSfuRoute, + type HostSfuPublisherSnapshot, +} from "../media/host-sfu-route"; import { ParentEdgeQualityEvidenceReporter, } from "../media/parent-edge-quality-evidence"; @@ -232,6 +235,8 @@ export function HostPage({ onAuthorizationRequired }: HostPageProps = {}) { const [peerSnapshots, setPeerSnapshots] = useState>( () => new Map(), ); + const [sfuPublisherSnapshot, setSfuPublisherSnapshot] = + useState(null); const [participantPresence, setParticipantPresence] = useState< ParticipantPresenceEntry[] >([]); @@ -406,6 +411,14 @@ export function HostPage({ onAuthorizationRequired }: HostPageProps = {}) { isCurrentGeneration(generation) && hostSfuRouteRef.current === route ? signalRef.current?.send(message) === true : false, + onPublisherUpdate: (snapshot) => { + if ( + isCurrentGeneration(generation) && + hostSfuRouteRef.current === route + ) { + setSfuPublisherSnapshot(snapshot); + } + }, }); hostSfuRouteRef.current = route; return route; @@ -414,6 +427,7 @@ export function HostPage({ onAuthorizationRequired }: HostPageProps = {}) { function clearHostSfuRoute(): void { const route = hostSfuRouteRef.current; hostSfuRouteRef.current = null; + setSfuPublisherSnapshot(null); sfuStandbyPrewarmerRef.current?.setUrl(null); void route?.disconnect(); } @@ -466,6 +480,7 @@ export function HostPage({ onAuthorizationRequired }: HostPageProps = {}) { hostPeerIdRef.current = null; void hostSfuRouteRef.current?.disconnect(); hostSfuRouteRef.current = null; + setSfuPublisherSnapshot(null); sfuStandbyPrewarmerRef.current?.setUrl(null); streamRef.current?.getTracks().forEach((track) => track.stop()); retiringStreamRef.current?.getTracks().forEach((track) => track.stop()); @@ -2315,6 +2330,21 @@ export function HostPage({ onAuthorizationRequired }: HostPageProps = {}) { )}
+ {showConnectionDetails && sfuPublisherSnapshot && ( +
+
+
+

SFU 发送

+ +
+
+ +
+ )} {viewers.map((viewer) => { const snapshot = viewer.upstream.kind === "peer" && diff --git a/src/client/sfu/publisher.ts b/src/client/sfu/publisher.ts index b267a75..e15f1a0 100644 --- a/src/client/sfu/publisher.ts +++ b/src/client/sfu/publisher.ts @@ -17,6 +17,13 @@ import { type ScreenAudioQuality, type VideoSenderParameterReadback, } from "../media/quality"; +import type { ConnectionMetrics } from "../types"; +import { + captureMetrics, + collectConnectionMetricsFromReport, + createStatsAccumulator, + type StatsAccumulator, +} from "../webrtc/stats"; export interface SfuConnectionConfig { url: string; @@ -27,6 +34,7 @@ export interface SfuConnectionConfig { interface PublisherEvents { onDisconnected?: () => void; + onStats?: (metrics: ConnectionMetrics | null) => void; } export type SfuPublisherFailureStage = @@ -47,6 +55,13 @@ interface PublishedVideoConfiguration { warning: string | null; } +interface PublisherStatsIdentity { + video: PublishedTrack; + sender: RTCRtpSender; + rawTrack: MediaStreamTrack; + accumulator: StatsAccumulator; +} + type LiveKit = typeof import("livekit-client"); type PublisherState = | "idle" @@ -56,6 +71,7 @@ type PublisherState = | "disconnected"; const roomDisconnects = new WeakMap>(); +const STATS_INTERVAL_MS = 2_000; export class SfuPublisher { private room: Room | null = null; @@ -70,6 +86,10 @@ export class SfuPublisher { private generation = 0; private operationTail: Promise = Promise.resolve(); private terminalNotified = false; + private statsVideo: PublishedTrack | null = null; + private statsIdentity: PublisherStatsIdentity | null = null; + private statsInFlight: PublisherStatsIdentity | null = null; + private statsTimer: ReturnType | null = null; constructor(private readonly events: PublisherEvents = {}) {} @@ -184,6 +204,7 @@ export class SfuPublisher { this.retainSenderParameters(videoConfiguration); this.failureStage = null; this.state = "active"; + this.startStats(video); return true; } catch (error) { if (this.owns(room, generation)) { @@ -205,6 +226,7 @@ export class SfuPublisher { if (this.state === "prepared") { return true; } + this.stopStats(); const video = this.video; const audio = this.audio; @@ -257,6 +279,7 @@ export class SfuPublisher { const previousAudio = this.audio; let videoReplaceAttempted = false; let audioReplaceAttempted = false; + this.stopStats(); try { videoReplaceAttempted = true; @@ -304,6 +327,7 @@ export class SfuPublisher { previousAudio.rawTrack = nextAudioTrack; } this.retainSenderParameters(videoConfiguration); + this.startStats(previousVideo); return true; } catch (error) { if (!this.owns(room, generation)) { @@ -350,6 +374,7 @@ export class SfuPublisher { failureWarning, videoConfiguration.warning, ); + this.startStats(previousVideo); } return false; } catch (rollbackError) { @@ -381,6 +406,7 @@ export class SfuPublisher { ) { return false; } + this.stopStats(); try { const videoConfiguration = await configurePublishedVideo( @@ -394,6 +420,7 @@ export class SfuPublisher { } this.profile = profile; this.retainSenderParameters(videoConfiguration); + this.startStats(video); return true; } catch (error) { if (!this.owns(room, generation)) { @@ -418,6 +445,7 @@ export class SfuPublisher { failureWarning, videoConfiguration.warning, ); + this.startStats(video); return false; } catch (rollbackError) { if (this.owns(room, generation)) { @@ -489,6 +517,7 @@ export class SfuPublisher { return null; } ++this.generation; + this.stopStats(); this.state = "disconnected"; const room = this.room; this.room = null; @@ -524,6 +553,96 @@ export class SfuPublisher { this.senderParameters = configuration.readback; this.qualityWarning = configuration.warning; } + + private startStats(video: PublishedTrack): void { + this.stopStats(); + this.statsVideo = video; + this.statsTimer = setInterval(() => { + void this.updateStats(video); + }, STATS_INTERVAL_MS); + } + + private async updateStats(video: PublishedTrack): Promise { + if ( + this.state !== "active" || + this.statsVideo !== video || + this.video !== video + ) { + return; + } + const sender = video.publication.videoTrack?.sender; + if (!sender || sender.track !== video.rawTrack) { + this.resetStatsIdentity(); + return; + } + let identity = this.statsIdentity; + if ( + !identity || + identity.video !== video || + identity.sender !== sender || + identity.rawTrack !== video.rawTrack + ) { + this.resetStatsIdentity(); + identity = { + video, + sender, + rawTrack: video.rawTrack, + accumulator: createStatsAccumulator(), + }; + this.statsIdentity = identity; + } + if (this.statsInFlight === identity) { + return; + } + this.statsInFlight = identity; + try { + const report = await identity.sender.getStats(); + if ( + this.state !== "active" || + this.statsIdentity !== identity || + this.video !== identity.video || + identity.video.rawTrack !== identity.rawTrack || + identity.video.publication.videoTrack?.sender !== identity.sender || + identity.sender.track !== identity.rawTrack + ) { + return; + } + const metrics = { + ...collectConnectionMetricsFromReport( + report, + "send", + identity.accumulator, + { trackIdentifier: identity.rawTrack.id, rid: "h" }, + ), + ...captureMetrics(identity.rawTrack), + }; + this.events.onStats?.(metrics); + } catch { + // Stats are observational and must never disrupt active SFU media. + } finally { + if (this.statsInFlight === identity) { + this.statsInFlight = null; + } + } + } + + private stopStats(): void { + if (this.statsTimer !== null) { + clearInterval(this.statsTimer); + this.statsTimer = null; + } + this.statsVideo = null; + this.statsInFlight = null; + this.resetStatsIdentity(); + } + + private resetStatsIdentity(): void { + const hadIdentity = this.statsIdentity !== null; + this.statsIdentity = null; + if (hadIdentity) { + this.events.onStats?.(null); + } + } } async function publishTrack( diff --git a/src/client/types.ts b/src/client/types.ts index feab37b..6db01ac 100644 --- a/src/client/types.ts +++ b/src/client/types.ts @@ -12,16 +12,22 @@ export interface ConnectionMetrics { rtpStatsId: string | null; rtpSsrc: number | null; rtpMid: string | null; + rtpRid: string | null; trackIdentifier: string | null; selectedCandidatePairId: string | null; captureWidth: number | null; captureHeight: number | null; captureFramesPerSecond: number | null; + mediaSourceFramesPerSecond: number | null; path: MediaPath; iceProtocol: string | null; localRelayProtocol: string | null; localCandidateType: string | null; remoteCandidateType: string | null; + localCandidateAddress: string | null; + localCandidatePort: number | null; + remoteCandidateAddress: string | null; + remoteCandidatePort: number | null; rttMs: number | null; bitrateKbps: number | null; availableOutgoingKbps: number | null; @@ -48,6 +54,11 @@ export interface ConnectionMetrics { audioBitrateKbps: number | null; audioPacketLossPercent: number | null; audioJitterMs: number | null; + audioVideoPlayoutDeltaMs: number | null; + videoJitterBufferDelayMs: number | null; + audioJitterBufferDelayMs: number | null; + audioConcealedSamplesPercent: number | null; + intervalAudioConcealmentEvents: number | null; audioCodec: string | null; audioCodecClockRate: number | null; audioCodecChannels: number | null; @@ -55,6 +66,8 @@ export interface ConnectionMetrics { scalabilityMode: string | null; encoderImplementation: string | null; powerEfficientEncoder: boolean | null; + intervalFramesEncoded: number | null; + intervalEncodeTimeMs: number | null; intervalEncodeMs: number | null; intervalDecodeMs: number | null; qualityLimitationReason: string | null; @@ -77,16 +90,22 @@ export const EMPTY_METRICS: ConnectionMetrics = { rtpStatsId: null, rtpSsrc: null, rtpMid: null, + rtpRid: null, trackIdentifier: null, selectedCandidatePairId: null, captureWidth: null, captureHeight: null, captureFramesPerSecond: null, + mediaSourceFramesPerSecond: null, path: "unknown", iceProtocol: null, localRelayProtocol: null, localCandidateType: null, remoteCandidateType: null, + localCandidateAddress: null, + localCandidatePort: null, + remoteCandidateAddress: null, + remoteCandidatePort: null, rttMs: null, bitrateKbps: null, availableOutgoingKbps: null, @@ -113,6 +132,11 @@ export const EMPTY_METRICS: ConnectionMetrics = { audioBitrateKbps: null, audioPacketLossPercent: null, audioJitterMs: null, + audioVideoPlayoutDeltaMs: null, + videoJitterBufferDelayMs: null, + audioJitterBufferDelayMs: null, + audioConcealedSamplesPercent: null, + intervalAudioConcealmentEvents: null, audioCodec: null, audioCodecClockRate: null, audioCodecChannels: null, @@ -120,6 +144,8 @@ export const EMPTY_METRICS: ConnectionMetrics = { scalabilityMode: null, encoderImplementation: null, powerEfficientEncoder: null, + intervalFramesEncoded: null, + intervalEncodeTimeMs: null, intervalEncodeMs: null, intervalDecodeMs: null, qualityLimitationReason: null, diff --git a/src/client/webrtc/host-peer.ts b/src/client/webrtc/host-peer.ts index 94dd546..e9e05a5 100644 --- a/src/client/webrtc/host-peer.ts +++ b/src/client/webrtc/host-peer.ts @@ -9,10 +9,10 @@ import { } from "../media/quality"; import { EMPTY_METRICS, - type ConnectionMetrics, type PeerSnapshot, } from "../types"; import { + captureMetrics, collectConnectionMetrics, createStatsAccumulator, } from "./stats"; @@ -30,38 +30,6 @@ interface HostPeerEvents { onUpdate: (snapshot: PeerSnapshot) => void; } -type CaptureMetrics = Pick< - ConnectionMetrics, - "captureWidth" | "captureHeight" | "captureFramesPerSecond" ->; - -function captureMetrics(track: MediaStreamTrack): CaptureMetrics { - if (typeof track.getSettings !== "function") { - return { - captureWidth: null, - captureHeight: null, - captureFramesPerSecond: null, - }; - } - - try { - const settings = track.getSettings(); - const finiteNumber = (value: unknown): number | null => - typeof value === "number" && Number.isFinite(value) ? value : null; - return { - captureWidth: finiteNumber(settings.width), - captureHeight: finiteNumber(settings.height), - captureFramesPerSecond: finiteNumber(settings.frameRate), - }; - } catch { - return { - captureWidth: null, - captureHeight: null, - captureFramesPerSecond: null, - }; - } -} - export class HostPeer { readonly connectionId: string; diff --git a/src/client/webrtc/stats.ts b/src/client/webrtc/stats.ts index 556a3d6..2a6fec9 100644 --- a/src/client/webrtc/stats.ts +++ b/src/client/webrtc/stats.ts @@ -28,6 +28,8 @@ export interface StatsAccumulator { previousTotalFreezesDuration: number | null; previousRetransmittedPackets: number | null; previousRetransmittedBytes: number | null; + previousVideoJitterBufferDelay: number | null; + previousVideoJitterBufferEmittedCount: number | null; audioMediaId: string | null; audioSsrc: number | null; audioTrackIdentifier: string | null; @@ -36,10 +38,40 @@ export interface StatsAccumulator { audioTimestamp: number | null; previousAudioPacketsReceived: number | null; previousAudioPacketsLost: number | null; + previousAudioJitterBufferDelay: number | null; + previousAudioJitterBufferEmittedCount: number | null; + previousAudioTotalSamplesReceived: number | null; + previousAudioConcealedSamples: number | null; + previousAudioConcealmentEvents: number | null; } export interface StatsMediaSelector { trackIdentifier: string | null; + rid?: string | null; +} + +export function captureMetrics( + track: MediaStreamTrack, +): Pick< + ConnectionMetrics, + "captureWidth" | "captureHeight" | "captureFramesPerSecond" +> { + try { + const settings = track.getSettings(); + const finite = (value: unknown): number | null => + typeof value === "number" && Number.isFinite(value) ? value : null; + return { + captureWidth: finite(settings.width), + captureHeight: finite(settings.height), + captureFramesPerSecond: finite(settings.frameRate), + }; + } catch { + return { + captureWidth: null, + captureHeight: null, + captureFramesPerSecond: null, + }; + } } export function createStatsAccumulator(): StatsAccumulator { @@ -61,6 +93,8 @@ export function createStatsAccumulator(): StatsAccumulator { previousTotalFreezesDuration: null, previousRetransmittedPackets: null, previousRetransmittedBytes: null, + previousVideoJitterBufferDelay: null, + previousVideoJitterBufferEmittedCount: null, audioMediaId: null, audioSsrc: null, audioTrackIdentifier: null, @@ -69,6 +103,11 @@ export function createStatsAccumulator(): StatsAccumulator { audioTimestamp: null, previousAudioPacketsReceived: null, previousAudioPacketsLost: null, + previousAudioJitterBufferDelay: null, + previousAudioJitterBufferEmittedCount: null, + previousAudioTotalSamplesReceived: null, + previousAudioConcealedSamples: null, + previousAudioConcealmentEvents: null, }; } @@ -138,6 +177,24 @@ export function packetLossPercentFromDeltas( return packetDelta > 0 ? (packetsLostDelta / packetDelta) * 100 : null; } +function percentOfInterval( + partDelta: number | null, + totalDelta: number | null, +): number | null { + if ( + partDelta === null || + totalDelta === null || + !Number.isFinite(partDelta) || + !Number.isFinite(totalDelta) || + partDelta < 0 || + totalDelta <= 0 || + partDelta > totalDelta + ) { + return null; + } + return (partDelta / totalDelta) * 100; +} + function numberValue(record: StatsRecord | null, key: string): number | null { const value = record?.[key]; return typeof value === "number" && Number.isFinite(value) ? value : null; @@ -168,9 +225,17 @@ function mediaTrackIdentifier( if (direction === "receive") { return stringValue(media, "trackIdentifier"); } + const source = mediaSourceRecord(report, media); + return stringValue(source, "trackIdentifier"); +} + +function mediaSourceRecord( + report: RTCStatsReport, + media: StatsRecord | null, +): StatsRecord | null { const source = getRecord(report, stringValue(media, "mediaSourceId")); return source?.type === "media-source" - ? stringValue(source, "trackIdentifier") + ? source : null; } @@ -194,13 +259,14 @@ function mediaRecord( } }); const selectedTrackIdentifier = selector?.trackIdentifier ?? null; - const narrowed = selectedTrackIdentifier !== null - ? candidates.filter( - (candidate) => - mediaTrackIdentifier(report, candidate, direction) === - selectedTrackIdentifier, - ) - : candidates; + const selectedRid = selector?.rid ?? null; + const narrowed = candidates.filter( + (candidate) => + (selectedTrackIdentifier === null || + mediaTrackIdentifier(report, candidate, direction) === + selectedTrackIdentifier) && + (selectedRid === null || stringValue(candidate, "rid") === selectedRid), + ); return narrowed.length === 1 ? narrowed[0]! : null; } @@ -288,6 +354,25 @@ function positiveIntegerValue( return value !== null && Number.isInteger(value) && value > 0 ? value : null; } +function candidateAddressValue(record: StatsRecord | null): string | null { + const value = stringValue(record, "address"); + return value !== null && + value.length <= 255 && + !/[\u0000-\u0020\u007f]/.test(value) + ? value + : null; +} + +function candidatePortValue(record: StatsRecord | null): number | null { + const value = numberValue(record, "port"); + return value !== null && + Number.isInteger(value) && + value > 0 && + value <= 65_535 + ? value + : null; +} + function audioCodecParameters(codec: StatsRecord | null): string | null { const value = stringValue(codec, "sdpFmtpLine"); return value !== null && value.length <= 512 && /^[\x20-\x7e]+$/.test(value) @@ -393,6 +478,11 @@ export function collectConnectionMetricsFromReport( ? "retransmittedBytesSent" : "retransmittedBytesReceived", ); + const videoJitterBufferDelay = numberValue(media, "jitterBufferDelay"); + const videoJitterBufferEmittedCount = numberValue( + media, + "jitterBufferEmittedCount", + ); let bitrateKbps: number | null = null; let derivedFps: number | null = null; const sampleWindowMs = @@ -425,6 +515,18 @@ export function collectConnectionMetricsFromReport( sampleWindowMs !== null ? previous.frames : null, ) : null; + const intervalFramesEncoded = + direction === "send" + ? intervalDelta(framesEncoded, previous.frames, sampleWindowMs !== null) + : null; + const encodeTimeDelta = + direction === "send" + ? intervalDelta( + totalEncodeTime, + previous.previousTotalEncodeTime, + sampleWindowMs !== null, + ) + : null; const intervalDecodeMs = direction === "receive" ? intervalAverageMs( @@ -490,6 +592,15 @@ export function collectConnectionMetricsFromReport( previous.previousRetransmittedBytes, sampleWindowMs !== null, ); + const videoJitterBufferDelayMs = + direction === "receive" && sampleWindowMs !== null + ? intervalAverageMs( + videoJitterBufferDelay, + previous.previousVideoJitterBufferDelay, + videoJitterBufferEmittedCount, + previous.previousVideoJitterBufferEmittedCount, + ) + : null; previous.mediaId = mediaId; previous.ssrc = ssrc; previous.trackIdentifier = trackIdentifier; @@ -507,9 +618,14 @@ export function collectConnectionMetricsFromReport( previous.previousTotalFreezesDuration = totalFreezesDuration; previous.previousRetransmittedPackets = retransmittedPackets; previous.previousRetransmittedBytes = retransmittedBytes; + previous.previousVideoJitterBufferDelay = videoJitterBufferDelay; + previous.previousVideoJitterBufferEmittedCount = + videoJitterBufferEmittedCount; const linkedCodec = linkedMediaCodec(report, media, transport, "video"); const codecEvidence = deriveCodecEvidence(linkedCodec); + const mediaSource = + direction === "send" ? mediaSourceRecord(report, media) : null; const audio = mediaRecord(report, direction, null, "audio"); const audioTransport = transportRecord(report, audio); const audioRemoteInbound = @@ -546,6 +662,14 @@ export function collectConnectionMetricsFromReport( : null; const audioPacketsReceived = numberValue(audioLossSource, "packetsReceived"); const audioPacketsLost = numberValue(audioLossSource, "packetsLost"); + const audioJitterBufferDelay = numberValue(audio, "jitterBufferDelay"); + const audioJitterBufferEmittedCount = numberValue( + audio, + "jitterBufferEmittedCount", + ); + const audioTotalSamplesReceived = numberValue(audio, "totalSamplesReceived"); + const audioConcealedSamples = numberValue(audio, "concealedSamples"); + const audioConcealmentEvents = numberValue(audio, "concealmentEvents"); const audioPacketsReceivedDelta = intervalDelta( audioPacketsReceived, previous.previousAudioPacketsReceived, @@ -563,6 +687,44 @@ export function collectConnectionMetricsFromReport( audioBytes >= previous.audioBytes ? ((audioBytes - previous.audioBytes) * 8) / audioSampleWindowMs : null; + const audioJitterBufferDelayMs = + direction === "receive" && audioSampleWindowMs !== null + ? intervalAverageMs( + audioJitterBufferDelay, + previous.previousAudioJitterBufferDelay, + audioJitterBufferEmittedCount, + previous.previousAudioJitterBufferEmittedCount, + ) + : null; + const intervalAudioTotalSamplesReceived = intervalDelta( + audioTotalSamplesReceived, + previous.previousAudioTotalSamplesReceived, + direction === "receive" && audioSampleWindowMs !== null, + ); + const intervalAudioConcealedSamples = intervalDelta( + audioConcealedSamples, + previous.previousAudioConcealedSamples, + direction === "receive" && audioSampleWindowMs !== null, + ); + const intervalAudioConcealmentEvents = intervalDelta( + audioConcealmentEvents, + previous.previousAudioConcealmentEvents, + direction === "receive" && audioSampleWindowMs !== null, + ); + const videoEstimatedPlayoutTimestamp = numberValue( + media, + "estimatedPlayoutTimestamp", + ); + const audioEstimatedPlayoutTimestamp = numberValue( + audio, + "estimatedPlayoutTimestamp", + ); + const audioVideoPlayoutDeltaMs = + direction === "receive" && + videoEstimatedPlayoutTimestamp !== null && + audioEstimatedPlayoutTimestamp !== null + ? audioEstimatedPlayoutTimestamp - videoEstimatedPlayoutTimestamp + : null; previous.audioMediaId = audioMediaId; previous.audioSsrc = audioSsrc; previous.audioTrackIdentifier = audioTrackIdentifier; @@ -571,6 +733,12 @@ export function collectConnectionMetricsFromReport( previous.audioTimestamp = audioTimestamp; previous.previousAudioPacketsReceived = audioPacketsReceived; previous.previousAudioPacketsLost = audioPacketsLost; + previous.previousAudioJitterBufferDelay = audioJitterBufferDelay; + previous.previousAudioJitterBufferEmittedCount = + audioJitterBufferEmittedCount; + previous.previousAudioTotalSamplesReceived = audioTotalSamplesReceived; + previous.previousAudioConcealedSamples = audioConcealedSamples; + previous.previousAudioConcealmentEvents = audioConcealmentEvents; const linkedAudioCodec = linkedMediaCodec( report, audio, @@ -592,6 +760,7 @@ export function collectConnectionMetricsFromReport( rtpStatsId: mediaId, rtpSsrc: ssrc, rtpMid: stringValue(media, "mid"), + rtpRid: stringValue(media, "rid"), trackIdentifier, selectedCandidatePairId: pair?.id ?? null, path, @@ -599,6 +768,10 @@ export function collectConnectionMetricsFromReport( localRelayProtocol, localCandidateType: localType, remoteCandidateType: remoteType, + localCandidateAddress: candidateAddressValue(localCandidate), + localCandidatePort: candidatePortValue(localCandidate), + remoteCandidateAddress: candidateAddressValue(remoteCandidate), + remoteCandidatePort: candidatePortValue(remoteCandidate), rttMs: numberValue(pair, "currentRoundTripTime") !== null ? numberValue(pair, "currentRoundTripTime")! * 1_000 @@ -610,6 +783,7 @@ export function collectConnectionMetricsFromReport( numberValue(pair, "availableOutgoingBitrate") !== null ? numberValue(pair, "availableOutgoingBitrate")! / 1_000 : null, + mediaSourceFramesPerSecond: numberValue(mediaSource, "framesPerSecond"), framesPerSecond: numberValue(media, "framesPerSecond") ?? derivedFps, frameWidth: width, frameHeight: height, @@ -647,6 +821,14 @@ export function collectConnectionMetricsFromReport( numberValue(audioLossSource, "jitter") !== null ? numberValue(audioLossSource, "jitter")! * 1_000 : null, + audioVideoPlayoutDeltaMs, + videoJitterBufferDelayMs, + audioJitterBufferDelayMs, + audioConcealedSamplesPercent: percentOfInterval( + intervalAudioConcealedSamples, + intervalAudioTotalSamplesReceived, + ), + intervalAudioConcealmentEvents, audioCodec: stringValue(linkedAudioCodec, "mimeType"), audioCodecClockRate: positiveIntegerValue(linkedAudioCodec, "clockRate"), audioCodecChannels: positiveIntegerValue(linkedAudioCodec, "channels"), @@ -655,6 +837,9 @@ export function collectConnectionMetricsFromReport( direction === "send" ? scalabilityModeValue(media) : null, encoderImplementation: stringValue(media, "encoderImplementation"), powerEfficientEncoder: booleanValue(media, "powerEfficientEncoder"), + intervalFramesEncoded, + intervalEncodeTimeMs: + encodeTimeDelta === null ? null : encodeTimeDelta * 1_000, intervalEncodeMs, intervalDecodeMs, qualityLimitationReason: stringValue(media, "qualityLimitationReason"), diff --git a/tests/client-state.test.ts b/tests/client-state.test.ts index 88a5ca5..f75b5d6 100644 --- a/tests/client-state.test.ts +++ b/tests/client-state.test.ts @@ -828,10 +828,14 @@ describe("WebRTC stats parsing", () => { entry("local-a", "local-candidate", { candidateType: "host", protocol: "udp", + address: "192.0.2.10", + port: 50_000, }), entry("remote-candidate-a", "remote-candidate", { candidateType: "srflx", protocol: "udp", + address: "2001:db8::10", + port: 50_001, }), entry("local-z", "local-candidate", { candidateType: "relay", @@ -849,9 +853,13 @@ describe("WebRTC stats parsing", () => { mediaSourceId: "source-a", remoteId: "remote-inbound-a", codecId: "codec-a", + rid: "main", scalabilityMode: "L2T3_KEY", bytesSent: 2_000, framesEncoded: 60, + totalEncodeTime: 0.3, + encoderImplementation: "ExternalEncoder", + powerEfficientEncoder: true, }), entry("outbound-z", "outbound-rtp", { kind: "video", @@ -866,6 +874,7 @@ describe("WebRTC stats parsing", () => { entry("source-a", "media-source", { kind: "video", trackIdentifier: "capture-track-a", + framesPerSecond: 58.5, }), entry("source-z", "media-source", { kind: "video", @@ -913,10 +922,16 @@ describe("WebRTC stats parsing", () => { rtpStatsId: "outbound-a", rtpSsrc: 111, rtpMid: "0", + rtpRid: "main", trackIdentifier: "capture-track-a", + mediaSourceFramesPerSecond: 58.5, selectedCandidatePairId: "pair-a", path: "direct", iceProtocol: "udp", + localCandidateAddress: "192.0.2.10", + localCandidatePort: 50_000, + remoteCandidateAddress: "2001:db8::10", + remoteCandidatePort: 50_001, packetsLost: 2, jitterMs: 4, rttMs: 20, @@ -925,10 +940,30 @@ describe("WebRTC stats parsing", () => { codecParameters: "packetization-mode=1; level-asymmetry-allowed=1", scalabilityMode: "L2T3_KEY", + encoderImplementation: "ExternalEncoder", + powerEfficientEncoder: true, + intervalFramesEncoded: null, + intervalEncodeTimeMs: null, }); expect(JSON.stringify(metrics)).not.toContain("sprop-parameter-sets"); expect(JSON.stringify(metrics)).not.toContain("x-google-start-bitrate"); + report.get("remote-candidate-a")!.address = null; + delete report.get("remote-candidate-a")!.port; + expect( + await collectConnectionMetrics(connection, "send", accumulator, { + trackIdentifier: "capture-track-a", + }), + ).toMatchObject({ + selectedCandidatePairId: "pair-a", + localCandidateAddress: "192.0.2.10", + localCandidatePort: 50_000, + remoteCandidateAddress: null, + remoteCandidatePort: null, + }); + report.get("remote-candidate-a")!.address = "2001:db8::10"; + report.get("remote-candidate-a")!.port = 50_001; + report.get("transport-a")!.selectedCandidatePairId = "pair-z"; const wrongTransportPair = await collectConnectionMetrics( connection, @@ -1773,6 +1808,178 @@ describe("WebRTC stats parsing", () => { }); }); + it("derives local inbound playout, jitter-buffer, and concealment evidence safely", async () => { + const report = ( + timestamp: number, + video: Record = {}, + audio: Record = {}, + ) => + new Map([ + [ + "video-inbound", + { + id: "video-inbound", + type: "inbound-rtp", + timestamp, + kind: "video", + ssrc: 101, + trackIdentifier: "shared-video", + bytesReceived: timestamp * 10, + framesDecoded: timestamp / 20, + ...video, + }, + ], + [ + "audio-inbound", + { + id: "audio-inbound", + type: "inbound-rtp", + timestamp, + kind: "audio", + ssrc: 202, + trackIdentifier: "shared-audio", + bytesReceived: timestamp * 2, + ...audio, + }, + ], + ]) as unknown as RTCStatsReport; + const reports = [ + report( + 1_000, + { + estimatedPlayoutTimestamp: 10_000, + jitterBufferDelay: 1.2, + jitterBufferEmittedCount: 60, + }, + { + estimatedPlayoutTimestamp: 10_012.5, + jitterBufferDelay: 960, + jitterBufferEmittedCount: 48_000, + totalSamplesReceived: 48_000, + concealedSamples: 480, + concealmentEvents: 2, + }, + ), + report( + 3_000, + { + estimatedPlayoutTimestamp: 12_000, + jitterBufferDelay: 4.2, + jitterBufferEmittedCount: 180, + }, + { + estimatedPlayoutTimestamp: 12_008.5, + jitterBufferDelay: 3_120, + jitterBufferEmittedCount: 144_000, + totalSamplesReceived: 144_000, + concealedSamples: 1_440, + concealmentEvents: 5, + }, + ), + report( + 5_000, + { + estimatedPlayoutTimestamp: 14_000, + jitterBufferDelay: 7.8, + jitterBufferEmittedCount: 300, + }, + { + ssrc: 303, + trackIdentifier: "replacement-audio", + estimatedPlayoutTimestamp: 13_997, + jitterBufferDelay: 2_000, + jitterBufferEmittedCount: 96_000, + totalSamplesReceived: 96_000, + concealedSamples: 5_000, + concealmentEvents: 30, + }, + ), + report( + 7_000, + { + estimatedPlayoutTimestamp: 16_000, + jitterBufferDelay: 1, + jitterBufferEmittedCount: 20, + }, + { + ssrc: 303, + trackIdentifier: "replacement-audio", + estimatedPlayoutTimestamp: 15_996, + jitterBufferDelay: 480, + jitterBufferEmittedCount: 24_000, + totalSamplesReceived: 24_000, + concealedSamples: 100, + concealmentEvents: 1, + }, + ), + report(9_000, {}, { ssrc: 303, trackIdentifier: "replacement-audio" }), + ]; + const connection = { + getStats: async () => reports.shift()!, + } as unknown as RTCPeerConnection; + const accumulator = createStatsAccumulator(); + + const first = await collectConnectionMetrics( + connection, + "receive", + accumulator, + ); + const stable = await collectConnectionMetrics( + connection, + "receive", + accumulator, + ); + const changedAudio = await collectConnectionMetrics( + connection, + "receive", + accumulator, + ); + const reset = await collectConnectionMetrics( + connection, + "receive", + accumulator, + ); + const unsupported = await collectConnectionMetrics( + connection, + "receive", + accumulator, + ); + + expect(first).toMatchObject({ + audioVideoPlayoutDeltaMs: 12.5, + videoJitterBufferDelayMs: null, + audioJitterBufferDelayMs: null, + audioConcealedSamplesPercent: null, + intervalAudioConcealmentEvents: null, + }); + expect(stable.audioVideoPlayoutDeltaMs).toBeCloseTo(8.5); + expect(stable.videoJitterBufferDelayMs).toBeCloseTo(25); + expect(stable.audioJitterBufferDelayMs).toBeCloseTo(22.5); + expect(stable.audioConcealedSamplesPercent).toBeCloseTo(1); + expect(stable.intervalAudioConcealmentEvents).toBe(3); + expect(changedAudio).toMatchObject({ + audioVideoPlayoutDeltaMs: -3, + audioJitterBufferDelayMs: null, + audioConcealedSamplesPercent: null, + intervalAudioConcealmentEvents: null, + }); + expect(changedAudio.videoJitterBufferDelayMs).toBeCloseTo(30); + expect(reset).toMatchObject({ + audioVideoPlayoutDeltaMs: -4, + videoJitterBufferDelayMs: null, + audioJitterBufferDelayMs: null, + audioConcealedSamplesPercent: null, + intervalAudioConcealmentEvents: null, + }); + expect(unsupported).toMatchObject({ + audioVideoPlayoutDeltaMs: null, + videoJitterBufferDelayMs: null, + audioJitterBufferDelayMs: null, + audioConcealedSamplesPercent: null, + intervalAudioConcealmentEvents: null, + }); + }); + it("uses linked remote inbound counters for outbound audio loss", async () => { const report = ( timestamp: number, diff --git a/tests/route-transition.test.ts b/tests/route-transition.test.ts index e6815d7..01c65fd 100644 --- a/tests/route-transition.test.ts +++ b/tests/route-transition.test.ts @@ -11,7 +11,10 @@ import { import { HostSfuRoute } from "../src/client/media/host-sfu-route.ts"; import { ViewerSfuRoute } from "../src/client/media/viewer-sfu-route.ts"; import { QUALITY_PROFILES } from "../src/client/media/quality.ts"; -import type { ConnectionMetrics } from "../src/client/types.ts"; +import { + EMPTY_METRICS, + type ConnectionMetrics, +} from "../src/client/types.ts"; const peerAssignment = ( parentPeerId: string, @@ -232,6 +235,76 @@ describe("MediaRouteTransition", () => { }); describe("HostSfuRoute", () => { + it("exposes only current active publication stats and clears retired evidence", async () => { + const updates: Array< + Parameters[0]["onPublisherUpdate"]>>[0] + > = []; + const statsCallbacks: Array<(metrics: ConnectionMetrics | null) => void> = []; + const publishers: ReturnType[] = []; + const route = new HostSfuRoute({ + getStream: () => ({}) as MediaStream, + getProfile: () => QUALITY_PROFILES["720p30"], + reconcileChildren: () => undefined, + send: () => true, + onPublisherUpdate: (snapshot) => updates.push(snapshot), + createPublisher: (_onDisconnected, onStats) => { + const publisher = createFakePublisher( + [], + `publisher-${publishers.length + 1}`, + ); + publishers.push(publisher); + statsCallbacks.push(onStats); + return publisher; + }, + }); + const metrics = { + ...EMPTY_METRICS, + trackIdentifier: "screen-a", + framesPerSecond: 60, + }; + const generationA = hostAssignment("generation-a"); + route.accept({ revision: 1, phase: "prepare", assignment: generationA }); + await route.acceptConfig(sfuConfig(1)); + await route.acceptAndWait({ + revision: 1, + phase: "active", + assignment: generationA, + }); + + statsCallbacks[0]?.(metrics); + expect(updates.at(-1)).toMatchObject({ + metrics: { trackIdentifier: "screen-a", framesPerSecond: 60 }, + }); + + const generationB = hostAssignment("generation-b"); + route.accept({ revision: 2, phase: "prepare", assignment: generationB }); + await route.acceptConfig(sfuConfig(2)); + await route.acceptAndWait({ + revision: 2, + phase: "active", + assignment: generationB, + }); + expect(updates.at(-1)).toBeNull(); + + statsCallbacks[0]?.({ ...metrics, framesPerSecond: 1 }); + expect(updates.at(-1)).toBeNull(); + statsCallbacks[1]?.({ + ...metrics, + trackIdentifier: "screen-b", + framesPerSecond: 30, + }); + expect(updates.at(-1)).toMatchObject({ + metrics: { trackIdentifier: "screen-b", framesPerSecond: 30 }, + }); + + await route.resyncAuthoritative({ + revision: 1, + phase: "active", + assignment: hostAssignment(null), + }); + expect(updates.at(-1)).toBeNull(); + }); + it("settles authoritative activation before exposing its quality warning", async () => { const publisher = { ...createFakePublisher([], "publisher"), diff --git a/tests/sfu-client.test.ts b/tests/sfu-client.test.ts index d7c4092..aea16b5 100644 --- a/tests/sfu-client.test.ts +++ b/tests/sfu-client.test.ts @@ -33,6 +33,7 @@ const livekit = vi.hoisted(() => { } class FakeSender { + track: MediaStreamTrack; parameters: RTCRtpSendParameters = { codecs: [], encodings: [{}], @@ -41,6 +42,10 @@ const livekit = vi.hoisted(() => { transactionId: "fake", }; + constructor(track: MediaStreamTrack) { + this.track = track; + } + readonly getParameters = vi.fn(() => this.parameters); readonly setParameters = vi.fn( async (parameters: RTCRtpSendParameters): Promise => { @@ -52,21 +57,27 @@ const livekit = vi.hoisted(() => { this.parameters = parameters; }, ); + readonly getStats = vi.fn( + async (): Promise => + new Map() as unknown as RTCStatsReport, + ); } class FakeLocalTrack { currentTrack: MediaStreamTrack; - readonly sender = new FakeSender(); + sender: FakeSender; publishOptions?: Record; savedDegradationPreference: RTCDegradationPreference | null = null; constructor(track: MediaStreamTrack) { this.currentTrack = track; + this.sender = new FakeSender(track); } readonly replaceTrack = vi.fn( async (nextTrack: MediaStreamTrack): Promise => { this.currentTrack = nextTrack; + this.sender.track = nextTrack; }, ); @@ -75,6 +86,11 @@ const livekit = vi.hoisted(() => { this.savedDegradationPreference = preference; }, ); + + replaceSenderForTest(): FakeSender { + this.sender = new FakeSender(this.currentTrack); + return this.sender; + } } class FakeLocalParticipant { @@ -302,7 +318,11 @@ const qualityProfile = { } as const; function track(kind: "video" | "audio", id: string): MediaStreamTrack { - return Object.assign(new EventTarget(), { id, kind }) as MediaStreamTrack; + return Object.assign(new EventTarget(), { + id, + kind, + getSettings: () => ({}), + }) as MediaStreamTrack; } function remoteTrack( @@ -332,6 +352,71 @@ function receiverReport(kind: "video" | "audio"): RTCStatsReport { timestamp: 1_000, kind, trackIdentifier: `${kind}-1`, + estimatedPlayoutTimestamp: kind === "video" ? 10_000 : 10_012, + }, + ]); +} + +function senderReport( + trackId: string, + timestamp: number, + bytesSent: number, + framesEncoded: number, +): RTCStatsReport { + return statsReport([ + { + id: "video-out-low", + type: "outbound-rtp", + timestamp, + kind: "video", + rid: "q", + mediaSourceId: "video-source", + bytesSent: Math.floor(bytesSent / 4), + framesEncoded: Math.floor(framesEncoded / 4), + framesPerSecond: 15, + frameWidth: 960, + frameHeight: 540, + totalEncodeTime: framesEncoded * 0.002, + qualityLimitationReason: "none", + }, + { + id: "video-out", + type: "outbound-rtp", + timestamp, + kind: "video", + rid: "h", + transportId: "transport", + codecId: "codec", + mediaSourceId: "video-source", + bytesSent, + framesEncoded, + framesPerSecond: 57, + frameWidth: 1920, + frameHeight: 1080, + totalEncodeTime: framesEncoded * 0.004, + qualityLimitationReason: "bandwidth", + encoderImplementation: "ExternalEncoder", + powerEfficientEncoder: true, + }, + { + id: "video-source", + type: "media-source", + timestamp, + trackIdentifier: trackId, + framesPerSecond: 59, + }, + { + id: "transport", + type: "transport", + timestamp, + }, + { + id: "codec", + type: "codec", + timestamp, + transportId: "transport", + mimeType: "video/H264", + sdpFmtpLine: "profile-level-id=42e01f;packetization-mode=1", }, ]); } @@ -359,10 +444,101 @@ beforeEach(() => { }); afterEach(() => { + vi.clearAllTimers(); + vi.useRealTimers(); vi.unstubAllGlobals(); }); describe("SfuPublisher", () => { + it("samples only the current published sender and clears replaced evidence", async () => { + vi.useFakeTimers(); + const updates: Array = []; + const publisher = new SfuPublisher({ + onStats: (metrics) => updates.push(metrics), + }); + const previousVideo = track("video", "video-1"); + previousVideo.getSettings = () => ({ + width: 1920, + height: 1080, + frameRate: 60, + }); + await publisher.connect(connection); + await publisher.activate(stream(previousVideo), qualityProfile); + const sender = livekit.state.rooms[0].localParticipant.publications[0].track + .sender; + sender.getStats.mockResolvedValueOnce( + senderReport(previousVideo.id, 1_000, 100_000, 60), + ); + + await vi.advanceTimersByTimeAsync(2_000); + expect(updates.at(-1)).toMatchObject({ + captureWidth: 1920, + captureHeight: 1080, + captureFramesPerSecond: 60, + trackIdentifier: previousVideo.id, + rtpRid: "h", + mediaSourceFramesPerSecond: 59, + framesPerSecond: 57, + resolution: "1920x1080", + codec: "video/H264", + codecProfile: "profile-level-id=42e01f", + encoderImplementation: "ExternalEncoder", + powerEfficientEncoder: true, + qualityLimitationReason: "bandwidth", + }); + + sender.getStats.mockResolvedValueOnce( + senderReport(previousVideo.id, 3_000, 300_000, 180), + ); + await vi.advanceTimersByTimeAsync(2_000); + expect(updates.at(-1)).toMatchObject({ + bitrateKbps: 800, + intervalFramesEncoded: 120, + intervalEncodeTimeMs: 480, + intervalEncodeMs: 4, + }); + + let resolveOld!: (report: RTCStatsReport) => void; + sender.getStats.mockImplementationOnce( + () => new Promise((resolve) => { resolveOld = resolve; }), + ); + await vi.advanceTimersByTimeAsync(2_000); + const nextVideo = track("video", "video-2"); + nextVideo.getSettings = () => ({ + width: 1280, + height: 720, + frameRate: 30, + }); + await expect(publisher.replaceStream(stream(nextVideo))).resolves.toBe(true); + expect(updates.at(-1)).toBeNull(); + resolveOld(senderReport(previousVideo.id, 5_000, 500_000, 300)); + await Promise.resolve(); + expect(updates.at(-1)).toBeNull(); + + sender.getStats.mockResolvedValueOnce( + senderReport(nextVideo.id, 4_000, 400_000, 240), + ); + await vi.advanceTimersByTimeAsync(2_000); + expect(updates.at(-1)?.trackIdentifier).toBe(nextVideo.id); + + const updateCount = updates.length; + const localTrack = livekit.state.rooms[0].localParticipant.publications[0] + .track; + const replacementSender = localTrack.replaceSenderForTest(); + replacementSender.getStats.mockResolvedValueOnce( + senderReport(nextVideo.id, 6_000, 500_000, 270), + ); + await vi.advanceTimersByTimeAsync(2_000); + expect(updates[updateCount]).toBeNull(); + expect(updates.at(-1)).toMatchObject({ + trackIdentifier: nextVideo.id, + framesPerSecond: 57, + }); + + await publisher.deactivate(); + expect(updates.at(-1)).toBeNull(); + }); + it("prepares a connection without publishing media", async () => { const publisher = new SfuPublisher(); @@ -1270,6 +1446,7 @@ describe("SfuSubscriber", () => { expect(updates.at(-1)).toMatchObject({ rtpStatsId: "video-in", trackIdentifier: "video-1", + audioVideoPlayoutDeltaMs: 12, }); let releaseStats = (): void => undefined; diff --git a/tests/stats-grid.test.ts b/tests/stats-grid.test.ts index 0e86943..75ffa15 100644 --- a/tests/stats-grid.test.ts +++ b/tests/stats-grid.test.ts @@ -19,11 +19,22 @@ const metrics = { iceProtocol: "udp", localCandidateType: "srflx", remoteCandidateType: "host", + localCandidateAddress: "192.0.2.10", + localCandidatePort: 50_000, + remoteCandidateAddress: "2001:db8::10", + remoteCandidatePort: 50_001, codec: "video/VP8", codecParameters: "max-fs=8160", + captureWidth: 1920, + captureHeight: 1080, + captureFramesPerSecond: 60, + mediaSourceFramesPerSecond: 58.5, + rtpRid: "h", audioBitrateKbps: 128, audioCodec: "audio/opus", encoderImplementation: "ExternalEncoder", + intervalFramesEncoded: 116, + intervalEncodeTimeMs: 371.2, intervalEncodeMs: 3.2, qualityLimitationReason: "bandwidth", } satisfies ConnectionMetrics; @@ -60,9 +71,18 @@ describe("StatsGrid progressive disclosure", () => { 'hidden=""', ); expect(html.indexOf("质量状态")).toBeLessThan(panelStart); + expect(html.indexOf("本地候选地址")).toBeGreaterThan(panelStart); + expect(html.indexOf("远端候选地址")).toBeGreaterThan(panelStart); + expect(html).toContain("192.0.2.10:50000"); + expect(html).toContain("[2001:db8::10]:50001"); expect(html.indexOf("视频 Codec")).toBeGreaterThan(panelStart); expect(html.indexOf("音频发送码率")).toBeGreaterThan(panelStart); + expect(html.indexOf("编码输入帧率")).toBeGreaterThan(panelStart); + expect(html.indexOf("最近区间编码量")).toBeGreaterThan(panelStart); expect(html.indexOf("最近区间编码/帧")).toBeGreaterThan(panelStart); + expect(html).toContain("58.5 fps"); + expect(html).toContain("116 帧 · 371.2 ms"); + expect(html).toContain("当前 RID"); }); it("keeps the existing full grid when progressive disclosure is not requested", () => { @@ -75,4 +95,63 @@ describe("StatsGrid progressive disclosure", () => { expect(html).not.toContain("更多指标"); expect(html).not.toContain("aria-expanded"); }); + + it("shows local inbound playout and repair evidence only for receivers", () => { + const receiverMetrics = { + ...metrics, + audioVideoPlayoutDeltaMs: -12.5, + videoJitterBufferDelayMs: 24.5, + audioJitterBufferDelayMs: 18.5, + audioConcealedSamplesPercent: 1.25, + intervalAudioConcealmentEvents: 3, + } satisfies ConnectionMetrics; + const receiveHtml = renderToStaticMarkup( + createElement(StatsGrid, { + metrics: receiverMetrics, + direction: "receive", + }), + ); + const sendHtml = renderToStaticMarkup( + createElement(StatsGrid, { + metrics: receiverMetrics, + direction: "send", + }), + ); + const unavailableReceiveHtml = renderToStaticMarkup( + createElement(StatsGrid, { + metrics, + direction: "receive", + }), + ); + + for (const label of [ + "音视频播放差", + "视频抖动缓冲", + "音频抖动缓冲", + "音频补偿样本率", + "音频补偿事件", + ]) { + expect(receiveHtml).toContain(label); + expect(sendHtml).not.toContain(label); + expect(unavailableReceiveHtml).not.toContain(label); + } + expect(receiveHtml).toContain("-12.5 ms"); + expect(receiveHtml).toContain("1.3%"); + }); + + it("omits candidate endpoints when the browser withholds either field", () => { + const html = renderToStaticMarkup( + createElement(StatsGrid, { + metrics: { + ...metrics, + localCandidatePort: null, + remoteCandidateAddress: null, + }, + direction: "receive", + }), + ); + + expect(html).not.toContain("本地候选地址"); + expect(html).not.toContain("远端候选地址"); + }); }); diff --git a/tests/viewer-quality-evidence.test.ts b/tests/viewer-quality-evidence.test.ts index f4b48de..6d2938b 100644 --- a/tests/viewer-quality-evidence.test.ts +++ b/tests/viewer-quality-evidence.test.ts @@ -33,6 +33,11 @@ function receiveMetrics( intervalPacketsReceived: 1_500, intervalPacketsLost: 2, jitterMs: 3.5, + audioVideoPlayoutDeltaMs: 12.5, + videoJitterBufferDelayMs: 24, + audioJitterBufferDelayMs: 18, + audioConcealedSamplesPercent: 1, + intervalAudioConcealmentEvents: 3, intervalFramesDecoded: 120, intervalFramesDropped: 1, intervalDecodeMs: 2.4, @@ -100,6 +105,10 @@ describe("viewer quality evidence", () => { bitrateKbps: 200_000, intervalPacketsLost: -1, codecParameters: "raw fmtp; secret=value/with/slash", + localCandidateAddress: "192.0.2.10", + localCandidatePort: 50_000, + remoteCandidateAddress: "203.0.113.20", + remoteCandidatePort: 50_001, }), ); expect(bounded?.metrics).toMatchObject({ @@ -113,6 +122,12 @@ describe("viewer quality evidence", () => { expect(JSON.stringify(bounded)).not.toContain("must-not-leave-the-client"); expect(JSON.stringify(bounded)).not.toContain("scalabilityMode"); expect(JSON.stringify(bounded)).not.toContain("L3T3_KEY"); + expect(JSON.stringify(bounded)).not.toContain("audioVideoPlayoutDeltaMs"); + expect(JSON.stringify(bounded)).not.toContain("JitterBufferDelayMs"); + expect(JSON.stringify(bounded)).not.toContain("Concealed"); + expect(JSON.stringify(bounded)).not.toContain("ConcealmentEvents"); + expect(JSON.stringify(bounded)).not.toContain("192.0.2.10"); + expect(JSON.stringify(bounded)).not.toContain("203.0.113.20"); }); it("rate-limits, deduplicates, and resets sequence per connection", () => {