From b78e25b4fb0787201b7342f8a3659407b3413af8 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 21 Aug 2026 10:12:13 +0800 Subject: [PATCH] feat(web): add audio and loss diagnostics (#120) --- docs/project-memory.md | 2 +- docs/research/browser-screen-audio-quality.md | 45 +++- docs/status.md | 4 +- docs/方案设计.md | 3 +- docs/需求理解.md | 2 +- src/client/components/StatsGrid.tsx | 46 +++- src/client/components/connection-details.ts | 6 + src/client/media/viewer-quality-evidence.ts | 5 + src/client/types.ts | 16 ++ src/client/webrtc/stats.ts | 181 +++++++++++-- tests/client-state.test.ts | 246 +++++++++++++++++- tests/connection-details.test.ts | 12 +- tests/viewer-quality-evidence.test.ts | 1 + 13 files changed, 528 insertions(+), 41 deletions(-) diff --git a/docs/project-memory.md b/docs/project-memory.md index 1102112..01e899f 100644 --- a/docs/project-memory.md +++ b/docs/project-memory.md @@ -53,7 +53,7 @@ Last updated: 2026-08-21 - Chrome 151 synthetic topology/quality runs kept fanout 2/1 and decoding; one relay close recovered in 5.32 seconds. Control evidence only. - Deployed Web source uses LiveKit 2.22.0's two-layer `q,h` layout. Local Chrome 151/LiveKit 1.13.5 kept one SFU/UDP root with growing inbound/decoded/rendered counters, clean leaves and no TURN. Production SFU media, zero-child BWE, and calibrated C+B loss remain open. - C+B uses three hard-bad pairs, one-use samples and guarded intent/cooldown. Deployed admission rescue moves the oldest childless zero-capacity Host leaf below an unassigned one-slot relay in one revision; no score/timer/global rebalance. Deployment passed, but no real room triggered it. -- Web names/window audio are deployed; current source adds full-display system audio and the music hint. Native WGC/MF remains source-only: 203 rendered frames, 500 Opus packets, and matched PID/LUID `VideoEncode`. VP8 remains the Web default. +- Web names/window audio are deployed; source adds full-display system audio, music hint, interval AV loss% and audio codec/bitrate/jitter details. Native WGC/MF remains source-only: 203 rendered frames, 500 Opus packets, matched PID/LUID `VideoEncode`. New Web audio work is undeployed; VP8 remains default. ## Provisional Quality Targets diff --git a/docs/research/browser-screen-audio-quality.md b/docs/research/browser-screen-audio-quality.md index 2c41566..1cf8c61 100644 --- a/docs/research/browser-screen-audio-quality.md +++ b/docs/research/browser-screen-audio-quality.md @@ -19,9 +19,11 @@ The Web sender keeps the current minimal behavior: - mark returned audio tracks with `contentHint = "music"`; - treat an audio track as optional and warn before publishing when none exists; - preserve returned audio through source changes, picture pause, P2P, relay, and - configured SFU routes; and + configured SFU routes; - expose no channel-count, sample-rate, Opus bitrate, stereo, DTX, or FEC - quality control. + quality control; and +- expose local, read-only audio RTP diagnostics without treating negotiated + codec fields as source-quality facts. The music hint is not a codec or quality mode. No codec-quality control is justified yet. In particular, Screener will not add @@ -128,7 +130,7 @@ behavior. ## Reference Implementations No code was copied. All repositories were inspected at pinned commits on -2026-08-19. +2026-08-19 and their cited source pages were rechecked on 2026-08-21. - LiveKit client-sdk-js, Apache-2.0, commit [`0a2110d`](https://github.com/livekit/client-sdk-js/tree/0a2110d39904a06722a0c4d1ddbb9390bb06ad4d): @@ -155,19 +157,39 @@ The Apache and MIT implementations support the same conclusion: a library can offer product-specific audio modes, but the stereo/DTX/FEC/Opus-fmtp portion is not equivalent to a portable browser sender control. -## Honest Future Diagnostics +Discord and Oopz have a narrower evidence boundary: -If audio diagnostics become necessary, keep four layers separate: +- Discord's official Go Live architecture says its native client captures the + selected process and child-process audio with OS-specific APIs and sends + audio and video in separate RTP packets. Discord's public Voice protocol + requires Opus at the protocol boundary and distinguishes a `Soundshare` + speaking bit, but neither source publishes the exact negotiated/runtime + audio parameters for each consumer Go Live session. This is useful native + product evidence, not a preset Screener can copy or an excuse to infer codec + state without stats. +- The retained read-only + [Oopz 0.87.425 package inspection](./native-shared-encode-sender.md) found a native Agora + RTC library, while its public help center does not publish screen-share audio + codec, bitrate, stereo, DTX, or FEC settings. Those artifacts do not prove + the codec or parameters used by a live Oopz session. + +## Honest Diagnostics Contract + +Audio diagnostics keep four layers separate: 1. **Capture:** track absent/present, plus `channelCount` and `sampleRate` only when the browser returns numeric values. 2. **Requested/applied:** an RTP ceiling only if Screener actually calls `setParameters()`, followed by immediate `getParameters()` readback. -3. **Negotiated:** the in-use codec MIME type and bounded derived fmtp evidence - from `RTCCodecStats`, labelled as negotiated parameters rather than actual - stereo/DTX/FEC; raw SDP/fmtp is not uploaded or persisted. -4. **Observed:** interval audio bitrate from byte deltas and receiver FEC - counters when implemented; missing values remain unknown. +3. **Negotiated:** current local details follow the unique audio RTP object's + `codecId` to a same-transport `RTCCodecStats` and display MIME type, clock + rate, channels and bounded fmtp as negotiated fields. Raw SDP/stats/fmtp are + not uploaded or persisted. +4. **Observed:** current local details derive interval audio bitrate from byte + deltas and interval loss from `lostDelta / (receivedDelta + lostDelta)`, and + display reported jitter. The first sample, changed identity, reset, negative + delta, zero denominator or absent field is unknown. Concealment, jitter + buffer and FEC counters remain an acceptance-matrix follow-up. Never substitute `48000`, `2`, `false`, or `0` for an unavailable field. @@ -273,6 +295,9 @@ Add a user-facing audio setting only when all of these are true: - [W3C WebCodecs Opus registration](https://www.w3.org/TR/webcodecs-opus-codec-registration/) - [RFC 7587: RTP Payload Format for Opus](https://www.rfc-editor.org/rfc/rfc7587.html) - [RFC 7874: WebRTC Audio Codec and Processing Requirements](https://www.rfc-editor.org/rfc/rfc7874.html) +- [Discord Go Live architecture](https://discord.com/blog/how-it-all-goes-live-an-overview-of-discords-streaming-technology) +- [Discord Voice protocol](https://docs.discord.com/developers/topics/voice-connections) +- [Oopz help center](https://help.oopz.cn/) - [MDN `getDisplayMedia()`](https://developer.mozilla.org/en-US/docs/Web/API/MediaDevices/getDisplayMedia) - [MDN `MediaTrackSettings.channelCount`](https://developer.mozilla.org/en-US/docs/Web/API/MediaTrackSettings/channelCount) - [MDN Browser Compatibility Data: `MediaDevices`](https://github.com/mdn/browser-compat-data/blob/main/api/MediaDevices.json) diff --git a/docs/status.md b/docs/status.md index 8fbe390..ed51469 100644 --- a/docs/status.md +++ b/docs/status.md @@ -23,7 +23,7 @@ Flagship; parallel; min run/smoke/rollback; benchmark later. Active UI/config/lo - After an answer, a generation-bound 15s initial-connect deadline enters ICE restart; success/replacement/disposal cancels it. It is deployed but not mobile-verified. - Production includes LiveKit's two-layer `q,h` sender guard; its functional media proof remains local, not production. - Room `1` C+B reparenting remains uncalibrated. Deployed admission rescue promotes an unassigned one-slot relay over the oldest childless zero-capacity Host leaf, with no scores or periodic optimization. -- Web roster has Host names. Source requests window/system audio by surface and uses the music hint; the full-display change awaits deploy. +- Web roster names are deployed. Source window/system audio, music hint, interval AV loss% and audio codec/bitrate/jitter details await deploy. ## Verified Evidence @@ -37,7 +37,7 @@ Flagship; parallel; min run/smoke/rollback; benchmark later. Active UI/config/lo - Access protocol/config/HTTP/storage/SQLite/signaling focused tests pass, including commit-first teardown and v1 rollback. - Native wire/session tests enforce at most two authoritative children, no presence-created edge, stale-revision ignore, and one bounded unsupported-route failure per revision. - Production includes initial/active selected Host ingress and peer last-mile; focused tests cover exactly-once retry, cleanup, rollback, and ordinary STUN-only. No relay media ran. -- Host names, the window-audio hint, all-room routing, and all-room Host selected ingress are active; the stale restart assertion is historical only. +- Host names/window audio/all-room routing/Host selected ingress are active. New AV stats pass 42 focused tests/typecheck; this is not audible proof. - Exact `16f6eab` passed typecheck, both builds and 30 files/445 tests. Deploy preserved all recorded config/network/SQLite hashes; DB, routes, services, endpoints, site access, and old-endpoint 404 passed. It triggered no TURN/SFU session. Rollback is exact `22119b907d3cf03ce8b06d6fb4596ce1a26fedd7`; `fd76277b05d491af8840b28f3132b7ff445d3cbe` remains secondary. ## Unverified Boundaries diff --git a/docs/方案设计.md b/docs/方案设计.md index 2693a74..da72811 100644 --- a/docs/方案设计.md +++ b/docs/方案设计.md @@ -300,8 +300,9 @@ hop 按[低服务器成本媒体路由](./research/low-server-media-routes.md) - `currentRoundTripTime`、发送码率和 `availableOutgoingBitrate`； - 实际帧率、分辨率和相邻有效采样窗的 RTP 丢包率； - codec、编码实现、最近采样区间内的每帧编解码耗时和 `qualityLimitationReason`；同一连接不并发采样，首个样本、无新帧、统计对象换代或计数器重置时显示未知并重新建立基线，不用连接生命周期均值掩盖后续过载。 +- 同一连接上实际存在且唯一的 audio RTP 对象所对应的区间码率、RTP 丢包率、jitter，以及由其 `codecId` 引用且绑定同一 transport 的 `RTCCodecStats` MIME、clock rate、channels 和最长 512 个可打印 ASCII 字符的 fmtp。clock/channels/fmtp 只标成 codec 协商事实，不推断捕获采样率、有效载荷立体声、DTX 或 FEC 当前行为。 -观看者显示接收码率、帧率、分辨率、RTP 丢包率、jitter、解码/丢帧和当前路径。丢包率只在同一 RTP generation 的相邻样本中以 `lostDelta / (receivedDelta + lostDelta)` 计算；首样本、对象/计数器重置、负 delta、零分母或字段缺失时显示“未知”，不把累计 `packetsLost` 伪装成百分比。RTT 只标为网络往返时间，不能冒充玻璃到玻璃延迟。 +观看者显示接收码率、帧率、分辨率、RTP 丢包率、jitter、解码/丢帧和当前路径；Host 对 outbound 使用其链接的 `remote-inbound-rtp` 计算接收侧丢包率。视频和音频丢包率都只在同一 RTP generation 的相邻样本中以 `lostDelta / (receivedDelta + lostDelta)` 计算；首样本、对象/计数器重置、负 delta、零分母或字段缺失时显示“未知”，不把累计 `packetsLost` 伪装成百分比。RTT 只标为网络往返时间，不能冒充玻璃到玻璃延迟。 ## 界面结构 diff --git a/docs/需求理解.md b/docs/需求理解.md index 9271b2f..3b9fae2 100644 --- a/docs/需求理解.md +++ b/docs/需求理解.md @@ -32,7 +32,7 @@ - [ ] 路径级质量状态：应用的 `HIGH`/`FALLBACK` 非对称窗口只决定拓扑资格和诊断，不命令正常的媒体层切换，也不计算综合分数。SFU BWE downshift 先进入 `suspect`；多个连续窗口的 sender/viewer 关联证据确认 `FALLBACK` 后，应用不得稳定保留该 parent，并把 downstream capacity 降为 0。计划或显式质量 fallback 仅是内建路径失败后的候选：已有 children 时先在 connection/assignment generation guard 下全部迁走，失败或代次过期则保持原 assignment/quality。Autonomous downshift 无法 packet-level 预先迁移，default-on 前必须验收 descendants 的短暂影响。只有更长的稳定恢复窗口和 cooldown 后才能重新声明 relay capacity。Viewer 请求必须鉴权、限频、去重，并由 sender 的 GCC/Transport-CC、实际码率、RTT、丢包、重传、编码限制、协商 codec/profile/parameters 与 viewer 解码、丢帧、冻结证据交叉验证；不能仅凭自报生效，也不得上传 raw SDP、candidate 地址或原始设备/网络标识。 - [ ] 断线恢复：临时网络变化时按不同层级做一次性恢复，而不是把同一动作机械重复三次。`failed` 边先做一次 ICE restart，再重建一次同 parent 的 `RTCPeerConnection`，随后尝试一个不同的合格 peer parent；只有 peer 路径耗尽才进入 SFU。`connecting`/`checking` 先显示不改变拓扑的慢连接状态，硬超时必须由真实移动网络证据确定，不能为了更快使用服务器而过早放弃最终可成功的 P2P。任一步恢复即停止后续动作；所有异步结果受 session、route revision、assignment/connection generation 约束。active SFU 只允许一次 fresh-token 重试，再失败即有界回到 peer baseline 或明确失败，并向 Host 显示不含 URL、地址、token 或原始错误的失败阶段。 - [ ] 可观测性：按目标 RTP transport 与不透明连接/ICE 代次关联 selected path、`iceGatheringState`/ICE/connection state、事件推导的建立时长、generation-baselined pair-change delta，以及 RTT、RTP 丢包率、抖动、码率、帧率、编码耗时和受限原因。候选与错误的 wire、日志和导出只使用[低服务器成本媒体路由](./research/low-server-media-routes.md)的脱敏字段和 code bucket，禁止 raw SDP、candidate、地址、URL、`errorText` 或网络标识。只有创建该 `RTCPeerConnection` 的本机可在折叠详情中显示浏览器对 selected local/remote candidate 实际暴露的地址与端口；该值不上传、不持久化、不用于地理/NAT 推断，缺失或被隐私保护时显示未知。 -- [ ] 渐进式诊断：分享者和观看者默认只看到房间、画面、核心状态和常用控制；候选路径、RTT、丢包、编解码器、编码耗时等技术详情由一个明确的“显示连接详情”开关展开。隐藏诊断不能隐藏失败原因、音频缺失、持续画质受限或需要用户操作的告警。 +- [ ] 渐进式诊断：分享者和观看者默认只看到房间、画面、核心状态和常用控制；候选路径、RTT、视频/音频 RTP 区间丢包率、实际编解码器、音频区间码率/jitter 和视频编码耗时等技术详情由一个明确的“显示连接详情”开关展开。音频 codec MIME、RTP clock、channels 与 fmtp 只能来自当前 RTP 对象引用的 `RTCCodecStats`，缺失即未知；`opus/48000/2` 不得写成 48 kHz 音源或实际立体声。隐藏诊断不能隐藏失败原因、音频缺失、持续画质受限或需要用户操作的告警。 - [ ] 站点访问：生产部署必须配置独立的 `SITE_ACCESS_PASSWORD` 控制建房、Host 信令角色和所有 code-only Viewer 入口；只有本地开发/测试可显式关闭。非空值接受 8 至 128 个可见 ASCII bytes，不能复用 TURN、LiveKit、TLS 或其他部署 secret。`POST /api/site-access` 验证成功签发 12 小时、无服务端会话状态的 HttpOnly、`SameSite=Strict` site-access cookie。有效 room-scoped fragment grant 可绕过站点 gate 直接进入对应房间；其余 `/r/{code}`、`/join` 和 public-watch 必须先持有效站点 cookie，之后才检查房间 public/private policy 和房间密码。失败统一且不得泄露房间是否存在或 policy；建房 HTTP 只认该 cookie。已移除的旧环境变量、endpoint 和 cookie 不保留兼容。不引入账号、JWT、服务端 session Map 或通用 logout。 - [ ] 房间级观看授权：新房间默认生成含 256-bit 随机量、限时且可撤销的 room-scoped Viewer grant；服务端只保存完整 grant 的 SHA-256 摘要。到期阻止以后鉴权/重连，但不为已鉴权 Viewer 设置踢出定时器；即时清退使用 rotate/revoke。浏览器从 URL fragment 严格读取 grant 后，只写该房间的 `sessionStorage` 并立即用 `history.replaceState` 清除 fragment；刷新和页面内重连可继续。私密房间缺失 grant 时必须先通过站点访问，再提供正确房间观看密码；任一缺失都 fail closed。grant 不得进入 `localStorage`、cookie、query、WebSocket URL、Referrer、日志或错误。Host 可 rotate 或 revoke；revoke 只锁定 grant 路径，已配置的房间密码仍是独立的后续进入方式。新 grant 摘要成功提交后必须清除 Viewer grace/路由、终止旧授权代次的信令和媒体并断开全部旧 Viewer，数据库写失败则不得改变连接。public-watch 是显式、可逆的 room policy，只放宽房间级观看，不绕过站点访问。 - [ ] 最小持久化：`ROOM_DATABASE_PATH` 可选且必须与 `SITE_ACCESS_PASSWORD` 同时配置。两者都存在时，SQLite 只持久保存从 `1` 开始自增的房间 ID、Host token 摘要、nullable Viewer grant 摘要，以及 nullable 48-byte Viewer-password material（16-byte salt + 32-byte scrypt verifier）；数据库 `CHECK` 约束非空 BLOB 长度，不新增用户、成员或 session 表。明文房间密码只短暂存在于当前页面和鉴权/派生调用内存，不进入 URL、cookie、浏览器持久存储、日志或 SQLite。持久房间停止后仍显示等待状态；私密邀请按期限或 Host rotate 更新。旧 schema 行在原子迁移时必须 fail-closed 为没有可用明文 grant 的锁定私密状态，Host rotate 后才能生成新邀请。 diff --git a/src/client/components/StatsGrid.tsx b/src/client/components/StatsGrid.tsx index 775ae0d..ebee2b0 100644 --- a/src/client/components/StatsGrid.tsx +++ b/src/client/components/StatsGrid.tsx @@ -1,5 +1,6 @@ import type { ConnectionMetrics } from "../types"; import type { VideoSenderParameterReadback } from "../media/quality"; +import { formatPacketLossPercent } from "./connection-details"; function readableNumber(value: number | null, digits = 0): string { return value === null || !Number.isFinite(value) ? "未知" : value.toFixed(digits); @@ -71,7 +72,10 @@ export function StatsGrid({ - + - + {metrics.codecProfile && ( - + )} {metrics.codecParameters && ( - + + )} + + + + + {metrics.audioCodecClockRate !== null && ( + + )} + {metrics.audioCodecChannels !== null && ( + + )} + {metrics.audioCodecParameters && ( + )} {direction === "send" ? ( <> diff --git a/src/client/components/connection-details.ts b/src/client/components/connection-details.ts index 507c07e..92f739d 100644 --- a/src/client/components/connection-details.ts +++ b/src/client/components/connection-details.ts @@ -1,5 +1,11 @@ import type { PeerSnapshot } from "../types"; +export function formatPacketLossPercent(value: number | null): string { + return value === null || !Number.isFinite(value) + ? "未知" + : `${value.toFixed(1)}%`; +} + export function qualityLimitationSummary( snapshots: readonly PeerSnapshot[], ): string | null { diff --git a/src/client/media/viewer-quality-evidence.ts b/src/client/media/viewer-quality-evidence.ts index 2f7973d..14cf9eb 100644 --- a/src/client/media/viewer-quality-evidence.ts +++ b/src/client/media/viewer-quality-evidence.ts @@ -13,6 +13,7 @@ import { type ConnectionMetrics, type PeerSnapshot, } from "../types"; +import { packetLossPercentFromDeltas } from "../webrtc/stats"; type ViewerQualityEvidence = Extract< ServerMessage, @@ -236,6 +237,10 @@ export function metricsFromQualityEvidence( packetsLost: metrics.packetsLostDelta, intervalPacketsReceived: metrics.packetsReceivedDelta, intervalPacketsLost: metrics.packetsLostDelta, + packetLossPercent: packetLossPercentFromDeltas( + metrics.packetsReceivedDelta, + metrics.packetsLostDelta, + ), jitterMs: metrics.jitterMs, framesDropped: metrics.framesDroppedDelta, intervalFramesDecoded: metrics.framesDecodedDelta, diff --git a/src/client/types.ts b/src/client/types.ts index ad29fac..feab37b 100644 --- a/src/client/types.ts +++ b/src/client/types.ts @@ -33,6 +33,7 @@ export interface ConnectionMetrics { intervalPacketsSent: number | null; intervalPacketsReceived: number | null; intervalPacketsLost: number | null; + packetLossPercent: number | null; jitterMs: number | null; framesDropped: number | null; intervalFramesDecoded: number | null; @@ -44,6 +45,13 @@ export interface ConnectionMetrics { codec: string | null; codecProfile: string | null; codecParameters: string | null; + audioBitrateKbps: number | null; + audioPacketLossPercent: number | null; + audioJitterMs: number | null; + audioCodec: string | null; + audioCodecClockRate: number | null; + audioCodecChannels: number | null; + audioCodecParameters: string | null; scalabilityMode: string | null; encoderImplementation: string | null; powerEfficientEncoder: boolean | null; @@ -90,6 +98,7 @@ export const EMPTY_METRICS: ConnectionMetrics = { intervalPacketsSent: null, intervalPacketsReceived: null, intervalPacketsLost: null, + packetLossPercent: null, jitterMs: null, framesDropped: null, intervalFramesDecoded: null, @@ -101,6 +110,13 @@ export const EMPTY_METRICS: ConnectionMetrics = { codec: null, codecProfile: null, codecParameters: null, + audioBitrateKbps: null, + audioPacketLossPercent: null, + audioJitterMs: null, + audioCodec: null, + audioCodecClockRate: null, + audioCodecChannels: null, + audioCodecParameters: null, scalabilityMode: null, encoderImplementation: null, powerEfficientEncoder: null, diff --git a/src/client/webrtc/stats.ts b/src/client/webrtc/stats.ts index dab2a05..f0c3e41 100644 --- a/src/client/webrtc/stats.ts +++ b/src/client/webrtc/stats.ts @@ -14,6 +14,7 @@ export interface StatsAccumulator { mediaId: string | null; ssrc: number | null; trackIdentifier: string | null; + lossSourceId: string | null; bytes: number | null; frames: number | null; timestamp: number | null; @@ -27,6 +28,14 @@ export interface StatsAccumulator { previousTotalFreezesDuration: number | null; previousRetransmittedPackets: number | null; previousRetransmittedBytes: number | null; + audioMediaId: string | null; + audioSsrc: number | null; + audioTrackIdentifier: string | null; + audioLossSourceId: string | null; + audioBytes: number | null; + audioTimestamp: number | null; + previousAudioPacketsReceived: number | null; + previousAudioPacketsLost: number | null; } export interface StatsMediaSelector { @@ -38,6 +47,7 @@ export function createStatsAccumulator(): StatsAccumulator { mediaId: null, ssrc: null, trackIdentifier: null, + lossSourceId: null, bytes: null, frames: null, timestamp: null, @@ -51,6 +61,14 @@ export function createStatsAccumulator(): StatsAccumulator { previousTotalFreezesDuration: null, previousRetransmittedPackets: null, previousRetransmittedBytes: null, + audioMediaId: null, + audioSsrc: null, + audioTrackIdentifier: null, + audioLossSourceId: null, + audioBytes: null, + audioTimestamp: null, + previousAudioPacketsReceived: null, + previousAudioPacketsLost: null, }; } @@ -92,6 +110,24 @@ function intervalAverageMs( : null; } +export function packetLossPercentFromDeltas( + packetsReceivedDelta: number | null, + packetsLostDelta: number | null, +): number | null { + if ( + packetsReceivedDelta === null || + packetsLostDelta === null || + !Number.isFinite(packetsReceivedDelta) || + !Number.isFinite(packetsLostDelta) || + packetsReceivedDelta < 0 || + packetsLostDelta < 0 + ) { + return null; + } + const packetDelta = packetsReceivedDelta + packetsLostDelta; + return packetDelta > 0 ? (packetsLostDelta / packetDelta) * 100 : null; +} + function numberValue(record: StatsRecord | null, key: string): number | null { const value = record?.[key]; return typeof value === "number" && Number.isFinite(value) ? value : null; @@ -132,11 +168,12 @@ function mediaRecord( report: RTCStatsReport, direction: "send" | "receive", selector: StatsMediaSelector | null, + kind: "audio" | "video" = "video", ): StatsRecord | null { const expectedType = direction === "send" ? "outbound-rtp" : "inbound-rtp"; const isExpectedMedia = (record: StatsRecord): boolean => record.type === expectedType && - record.kind === "video" && + record.kind === kind && record.isRemote !== true; const candidates: StatsRecord[] = []; @@ -200,17 +237,19 @@ function selectedCandidatePair( function linkedRemoteInbound( report: RTCStatsReport, outbound: StatsRecord | null, + kind: "audio" | "video" = "video", ): StatsRecord | null { const remote = getRecord(report, stringValue(outbound, "remoteId")); - return remote?.type === "remote-inbound-rtp" && remote.kind === "video" + return remote?.type === "remote-inbound-rtp" && remote.kind === kind ? remote : null; } -function linkedVideoCodec( +function linkedMediaCodec( report: RTCStatsReport, media: StatsRecord | null, transport: StatsRecord | null, + kind: "audio" | "video", ): StatsRecord | null { const codec = getRecord(report, stringValue(media, "codecId")); const mimeType = stringValue(codec, "mimeType"); @@ -218,7 +257,7 @@ function linkedVideoCodec( transport !== null && stringValue(codec, "transportId") === transport.id && mimeType !== null && - /^video\/[A-Za-z0-9.+-]{1,32}$/i.test(mimeType) + new RegExp(`^${kind}\\/[A-Za-z0-9.+-]{1,32}$`, "i").test(mimeType) ? codec : null; } @@ -231,6 +270,21 @@ function deriveCodecEvidence(codec: StatsRecord | null) { ); } +function positiveIntegerValue( + record: StatsRecord | null, + key: string, +): number | null { + const value = numberValue(record, key); + return value !== null && Number.isInteger(value) && value > 0 ? value : null; +} + +function audioCodecParameters(codec: StatsRecord | null): string | null { + const value = stringValue(codec, "sdpFmtpLine"); + return value !== null && value.length <= 512 && /^[\x20-\x7e]+$/.test(value) + ? value + : null; +} + function scalabilityModeValue(media: StatsRecord | null): string | null { const value = stringValue(media, "scalabilityMode"); return value && /^[A-Za-z0-9_-]{1,32}$/.test(value) ? value : null; @@ -276,13 +330,19 @@ export async function collectConnectionMetrics( const trackIdentifier = media ? mediaTrackIdentifier(report, media, direction) : null; + const remoteInbound = + direction === "send" ? linkedRemoteInbound(report, media) : null; + const lossSource = direction === "send" ? remoteInbound : media; + const lossSourceId = lossSource?.id ?? null; const sameMedia = mediaId !== null && mediaId === previous.mediaId && ssrc === previous.ssrc && trackIdentifier === previous.trackIdentifier; - const remoteInbound = - direction === "send" ? linkedRemoteInbound(report, media) : null; + const sameLossSource = + sameMedia && + lossSourceId !== null && + lossSourceId === previous.lossSourceId; const bytesKey = direction === "send" ? "bytesSent" : "bytesReceived"; const bytes = numberValue(media, bytesKey); const framesKey = direction === "send" ? "framesEncoded" : "framesDecoded"; @@ -291,11 +351,8 @@ export async function collectConnectionMetrics( const framesEncoded = numberValue(media, "framesEncoded"); const framesDecoded = numberValue(media, "framesDecoded"); const packetsSent = numberValue(media, "packetsSent"); - const packetsReceived = numberValue(media, "packetsReceived"); - const packetsLost = numberValue( - direction === "send" ? remoteInbound : media, - "packetsLost", - ); + const packetsReceived = numberValue(lossSource, "packetsReceived"); + const packetsLost = numberValue(lossSource, "packetsLost"); const totalEncodeTime = numberValue(media, "totalEncodeTime"); const totalDecodeTime = numberValue(media, "totalDecodeTime"); const framesDropped = numberValue(media, "framesDropped"); @@ -354,14 +411,13 @@ export async function collectConnectionMetrics( sampleWindowMs !== null ? previous.frames : null, ) : null; + const lossPacketsReceivedDelta = intervalDelta( + packetsReceived, + previous.previousPacketsReceived, + sampleWindowMs !== null && sameLossSource, + ); const intervalPacketsReceived = - direction === "receive" - ? intervalDelta( - packetsReceived, - previous.previousPacketsReceived, - sampleWindowMs !== null, - ) - : null; + direction === "receive" ? lossPacketsReceivedDelta : null; const intervalPacketsSent = direction === "send" ? intervalDelta( @@ -375,7 +431,7 @@ export async function collectConnectionMetrics( ? intervalDelta( packetsLost, previous.previousPacketsLost, - sampleWindowMs !== null, + sampleWindowMs !== null && sameLossSource, ) : null; const intervalFramesDecoded = @@ -414,6 +470,7 @@ export async function collectConnectionMetrics( previous.mediaId = mediaId; previous.ssrc = ssrc; previous.trackIdentifier = trackIdentifier; + previous.lossSourceId = lossSourceId; previous.bytes = bytes; previous.frames = frames; previous.timestamp = timestamp; @@ -428,8 +485,75 @@ export async function collectConnectionMetrics( previous.previousRetransmittedPackets = retransmittedPackets; previous.previousRetransmittedBytes = retransmittedBytes; - const linkedCodec = linkedVideoCodec(report, media, transport); + const linkedCodec = linkedMediaCodec(report, media, transport, "video"); const codecEvidence = deriveCodecEvidence(linkedCodec); + const audio = mediaRecord(report, direction, null, "audio"); + const audioTransport = transportRecord(report, audio); + const audioRemoteInbound = + direction === "send" + ? linkedRemoteInbound(report, audio, "audio") + : null; + const audioLossSource = direction === "send" ? audioRemoteInbound : audio; + const audioMediaId = audio?.id ?? null; + const audioSsrc = numberValue(audio, "ssrc"); + const audioTrackIdentifier = audio + ? mediaTrackIdentifier(report, audio, direction) + : null; + const audioLossSourceId = audioLossSource?.id ?? null; + const sameAudio = + audioMediaId !== null && + audioMediaId === previous.audioMediaId && + audioSsrc === previous.audioSsrc && + audioTrackIdentifier === previous.audioTrackIdentifier; + const sameAudioLossSource = + sameAudio && + audioLossSourceId !== null && + audioLossSourceId === previous.audioLossSourceId; + const audioBytes = numberValue( + audio, + direction === "send" ? "bytesSent" : "bytesReceived", + ); + const audioTimestamp = numberValue(audio, "timestamp"); + const audioSampleWindowMs = + sameAudio && + audioTimestamp !== null && + previous.audioTimestamp !== null && + audioTimestamp > previous.audioTimestamp + ? audioTimestamp - previous.audioTimestamp + : null; + const audioPacketsReceived = numberValue(audioLossSource, "packetsReceived"); + const audioPacketsLost = numberValue(audioLossSource, "packetsLost"); + const audioPacketsReceivedDelta = intervalDelta( + audioPacketsReceived, + previous.previousAudioPacketsReceived, + audioSampleWindowMs !== null && sameAudioLossSource, + ); + const audioPacketsLostDelta = intervalDelta( + audioPacketsLost, + previous.previousAudioPacketsLost, + audioSampleWindowMs !== null && sameAudioLossSource, + ); + const audioBitrateKbps = + audioSampleWindowMs !== null && + audioBytes !== null && + previous.audioBytes !== null && + audioBytes >= previous.audioBytes + ? ((audioBytes - previous.audioBytes) * 8) / audioSampleWindowMs + : null; + previous.audioMediaId = audioMediaId; + previous.audioSsrc = audioSsrc; + previous.audioTrackIdentifier = audioTrackIdentifier; + previous.audioLossSourceId = audioLossSourceId; + previous.audioBytes = audioBytes; + previous.audioTimestamp = audioTimestamp; + previous.previousAudioPacketsReceived = audioPacketsReceived; + previous.previousAudioPacketsLost = audioPacketsLost; + const linkedAudioCodec = linkedMediaCodec( + report, + audio, + audioTransport, + "audio", + ); const width = numberValue(media, "frameWidth"); const height = numberValue(media, "frameHeight"); const iceProtocol = @@ -471,6 +595,10 @@ export async function collectConnectionMetrics( intervalPacketsSent, intervalPacketsReceived, intervalPacketsLost, + packetLossPercent: packetLossPercentFromDeltas( + lossPacketsReceivedDelta, + intervalPacketsLost, + ), jitterMs: numberValue(direction === "send" ? remoteInbound : media, "jitter") !== null ? numberValue(direction === "send" ? remoteInbound : media, "jitter")! * @@ -487,6 +615,19 @@ export async function collectConnectionMetrics( codec: codecEvidence.codec, codecProfile: codecEvidence.profile, codecParameters: codecEvidence.parameters, + audioBitrateKbps, + audioPacketLossPercent: packetLossPercentFromDeltas( + audioPacketsReceivedDelta, + audioPacketsLostDelta, + ), + audioJitterMs: + numberValue(audioLossSource, "jitter") !== null + ? numberValue(audioLossSource, "jitter")! * 1_000 + : null, + audioCodec: stringValue(linkedAudioCodec, "mimeType"), + audioCodecClockRate: positiveIntegerValue(linkedAudioCodec, "clockRate"), + audioCodecChannels: positiveIntegerValue(linkedAudioCodec, "channels"), + audioCodecParameters: audioCodecParameters(linkedAudioCodec), scalabilityMode: direction === "send" ? scalabilityModeValue(media) : null, encoderImplementation: stringValue(media, "encoderImplementation"), diff --git a/tests/client-state.test.ts b/tests/client-state.test.ts index 9bd0c39..882a00a 100644 --- a/tests/client-state.test.ts +++ b/tests/client-state.test.ts @@ -1423,6 +1423,7 @@ describe("WebRTC stats parsing", () => { frameHeight: 1_080, intervalPacketsReceived: 1_500, intervalPacketsLost: 2, + packetLossPercent: (2 / 1_502) * 100, intervalFramesDecoded: 120, intervalDecodeMs: 2.5, codec: "video/H264", @@ -1431,6 +1432,249 @@ describe("WebRTC stats parsing", () => { scalabilityMode: null, }); expect(JSON.stringify(stable)).not.toContain("sprop-parameter-sets"); - expect(correctedLoss.intervalPacketsLost).toBeNull(); + expect(correctedLoss).toMatchObject({ + intervalPacketsLost: null, + packetLossPercent: null, + }); + }); + + it("reports verified inbound audio codec and interval transport facts", async () => { + const report = ( + timestamp: number, + videoPacketsReceived: number, + videoPacketsLost: number, + audioBytesReceived: number, + audioPacketsReceived: number, + audioPacketsLost: number, + audioCodecTransportId = "transport", + ) => + new Map([ + [ + "transport", + { id: "transport", type: "transport", timestamp }, + ], + [ + "video-codec", + { + id: "video-codec", + type: "codec", + timestamp, + transportId: "transport", + mimeType: "video/VP8", + }, + ], + [ + "audio-codec", + { + id: "audio-codec", + type: "codec", + timestamp, + transportId: audioCodecTransportId, + mimeType: "audio/opus", + clockRate: 48_000, + channels: 2, + sdpFmtpLine: "minptime=10;useinbandfec=1", + }, + ], + [ + "video-inbound", + { + id: "video-inbound", + type: "inbound-rtp", + timestamp, + kind: "video", + ssrc: 101, + transportId: "transport", + codecId: "video-codec", + bytesReceived: timestamp * 10, + packetsReceived: videoPacketsReceived, + packetsLost: videoPacketsLost, + framesDecoded: timestamp / 20, + }, + ], + [ + "audio-inbound", + { + id: "audio-inbound", + type: "inbound-rtp", + timestamp, + kind: "audio", + ssrc: 202, + trackIdentifier: "shared-audio", + transportId: "transport", + codecId: "audio-codec", + bytesReceived: audioBytesReceived, + packetsReceived: audioPacketsReceived, + packetsLost: audioPacketsLost, + jitter: 0.003, + }, + ], + ]) as unknown as RTCStatsReport; + const reports = [ + report(1_000, 100, 5, 10_000, 1_000, 5), + report(3_000, 190, 15, 50_000, 1_180, 10), + report(5_000, 190, 15, 50_000, 1_180, 10, "other-transport"), + ]; + const connection = { + getStats: async () => reports.shift()!, + } as unknown as RTCPeerConnection; + const accumulator = createStatsAccumulator(); + + const first = await collectConnectionMetrics( + connection, + "receive", + accumulator, + ); + const stable = await collectConnectionMetrics( + connection, + "receive", + accumulator, + ); + const zeroDenominator = await collectConnectionMetrics( + connection, + "receive", + accumulator, + ); + + expect(first).toMatchObject({ + packetLossPercent: null, + audioBitrateKbps: null, + audioPacketLossPercent: null, + audioJitterMs: 3, + audioCodec: "audio/opus", + audioCodecClockRate: 48_000, + audioCodecChannels: 2, + audioCodecParameters: "minptime=10;useinbandfec=1", + }); + expect(stable.packetLossPercent).toBeCloseTo(10); + expect(stable.audioBitrateKbps).toBeCloseTo(160); + expect(stable.audioPacketLossPercent).toBeCloseTo((5 / 185) * 100); + expect(zeroDenominator).toMatchObject({ + packetLossPercent: null, + audioBitrateKbps: 0, + audioPacketLossPercent: null, + audioCodec: null, + audioCodecClockRate: null, + audioCodecChannels: null, + audioCodecParameters: null, + }); + }); + + it("uses linked remote inbound counters for outbound audio loss", async () => { + const report = ( + timestamp: number, + audioBytesSent: number, + packetsReceived: number | undefined, + packetsLost: number | undefined, + ) => + new Map([ + [ + "transport", + { id: "transport", type: "transport", timestamp }, + ], + [ + "video-outbound", + { + id: "video-outbound", + type: "outbound-rtp", + timestamp, + kind: "video", + ssrc: 101, + bytesSent: timestamp * 10, + packetsSent: timestamp / 10, + framesEncoded: timestamp / 20, + }, + ], + [ + "audio-codec", + { + id: "audio-codec", + type: "codec", + timestamp, + transportId: "transport", + mimeType: "audio/opus", + clockRate: 48_000, + channels: 2, + }, + ], + [ + "audio-outbound", + { + id: "audio-outbound", + type: "outbound-rtp", + timestamp, + kind: "audio", + ssrc: 202, + transportId: "transport", + codecId: "audio-codec", + remoteId: "audio-remote-inbound", + bytesSent: audioBytesSent, + }, + ], + [ + "audio-remote-inbound", + { + id: "audio-remote-inbound", + type: "remote-inbound-rtp", + timestamp, + kind: "audio", + packetsReceived, + packetsLost, + jitter: 0.004, + }, + ], + ]) as unknown as RTCStatsReport; + const firstReport = report(1_000, 20_000, 500, 1); + (firstReport as unknown as Map).delete( + "audio-remote-inbound", + ); + const reports = [ + firstReport, + report(3_000, 50_000, 500, 1), + report(5_000, 80_000, 650, 4), + report(7_000, 110_000, 50, 1), + report(9_000, 140_000, 200, undefined), + ]; + const connection = { + getStats: async () => reports.shift()!, + } as unknown as RTCPeerConnection; + const accumulator = createStatsAccumulator(); + + await collectConnectionMetrics(connection, "send", accumulator); + const remoteAppeared = await collectConnectionMetrics( + connection, + "send", + accumulator, + ); + const stable = await collectConnectionMetrics( + connection, + "send", + accumulator, + ); + const reset = await collectConnectionMetrics( + connection, + "send", + accumulator, + ); + const missing = await collectConnectionMetrics( + connection, + "send", + accumulator, + ); + + expect(remoteAppeared).toMatchObject({ + audioBitrateKbps: 120, + audioPacketLossPercent: null, + }); + expect(stable).toMatchObject({ + audioBitrateKbps: 120, + audioJitterMs: 4, + audioCodec: "audio/opus", + audioCodecClockRate: 48_000, + audioCodecChannels: 2, + }); + expect(stable.audioPacketLossPercent).toBeCloseTo((3 / 153) * 100); + expect(reset.audioPacketLossPercent).toBeNull(); + expect(missing.audioPacketLossPercent).toBeNull(); }); }); diff --git a/tests/connection-details.test.ts b/tests/connection-details.test.ts index 260f5dd..53d4891 100644 --- a/tests/connection-details.test.ts +++ b/tests/connection-details.test.ts @@ -1,6 +1,9 @@ import { describe, expect, it } from "vitest"; -import { qualityLimitationSummary } from "../src/client/components/connection-details.ts"; +import { + formatPacketLossPercent, + qualityLimitationSummary, +} from "../src/client/components/connection-details.ts"; import { EMPTY_METRICS, type PeerSnapshot } from "../src/client/types.ts"; function snapshot(qualityWarning: string | null): PeerSnapshot { @@ -26,4 +29,11 @@ describe("progressive connection details", () => { ).toBe("持续受带宽限制，浏览器正在降低画面质量"); expect(qualityLimitationSummary([snapshot(null)])).toBeNull(); }); + + it("formats interval packet loss without inventing unknown values", () => { + expect(formatPacketLossPercent(1.25)).toBe("1.3%"); + expect(formatPacketLossPercent(0)).toBe("0.0%"); + expect(formatPacketLossPercent(null)).toBe("未知"); + expect(formatPacketLossPercent(Number.NaN)).toBe("未知"); + }); }); diff --git a/tests/viewer-quality-evidence.test.ts b/tests/viewer-quality-evidence.test.ts index dd56ff8..808224c 100644 --- a/tests/viewer-quality-evidence.test.ts +++ b/tests/viewer-quality-evidence.test.ts @@ -241,6 +241,7 @@ describe("viewer quality evidence", () => { resolution: "1920x1080", bitrateKbps: 7_500, packetsLost: 2, + packetLossPercent: (2 / 1_502) * 100, framesDropped: 1, intervalFramesDecoded: 120, intervalFramesDropped: 1,