From 5b66cb22a950c78e29b849b4984309cd87a3c208 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 19 Aug 2026 18:42:46 +0800 Subject: [PATCH] feat(media): expose negotiated codec evidence --- docs/project-memory.md | 4 +- docs/research/realtime-quality-adaptation.md | 51 ++++++- docs/status.md | 12 +- src/client/components/StatsGrid.tsx | 20 +++ src/client/media/quality.ts | 26 +++- src/client/types.ts | 6 + src/client/webrtc/stats.ts | 145 +++++++++++++++++- tests/client-state.test.ts | 146 +++++++++++++++++++ tests/host-peer-tracks.test.ts | 1 + tests/quality.test.ts | 72 ++++++++- tests/sfu-client.test.ts | 2 + 11 files changed, 462 insertions(+), 23 deletions(-) diff --git a/docs/project-memory.md b/docs/project-memory.md index 61ea8ba..b537f41 100644 --- a/docs/project-memory.md +++ b/docs/project-memory.md @@ -24,7 +24,7 @@ Last updated: 2026-08-19 - Keep ADR-0005 routing default-off: discrete failures drive peer/SFU recovery, full host fanout uses break-before-make, and mobile/iPad clients stay leaves. PR #20 prewarm passes locally; public transport/load gates remain. - Local reparenting is a later candidate: start with unassigned relay admission rescue; do not block current work. - Prefer direct UDP, then TURN/UDP, with TURN/TCP required; optional TURN/TLS defaults to 5349, while 443 needs a dedicated address or validated L4/SNI routing. -- Treat video settings as ceilings and degradation as unclassified. The first local A+B alignment foundation covers same-tick capture/outbound identity, windows, deltas, remote linkage, and path; next derive codec profile/parameters/`scalabilityMode`, then add minimal authenticated C, a two-state controller, and shared `LOW`. Never use UA or a composite score. +- Treat video settings as ceilings and degradation as unclassified. Local host A+B now covers same-tick capture/outbound identity, windows, deltas, remote linkage, path, and nullable negotiated codec/profile token/allowlisted parameters/current stream `scalabilityMode`. Use it first to reproduce the reported Host-only recovery, then add minimal authenticated C, a two-state controller, and shared `LOW`. Never use UA or a composite score. - Reported poor movie/video audio is unclassified. After video A+B, separately diagnose audio A/B/C and A/V sync across capture settings, codec/fmtp, actual bitrate, loss, jitter, concealment, and jitter buffer. `maxBitrate` is not quality-up; do not expand runtime before evidence. See `docs/research/browser-screen-audio-quality.md`. - Do not add custom scene detection or dynamic-FPS control until WebRTC statistics and host resource measurements prove a material gap. Keep browser codec order until target hardware proves a more efficient common codec. - Keep room policy deployment-driven: public/password-only rooms are random and temporary; password plus SQLite enables sequential persistent rooms and reusable stopped links. @@ -46,7 +46,7 @@ Last updated: 2026-08-19 - Chrome 151 short synthetic `1/3/5/8` runs kept host edges at two, relay edges at one, and all viewers decoding; a three-viewer relay close recovered in 5.32 seconds. This proves topology/control only, not quality, load, endurance, or silent partitions. - The current quality stack uses one strict, non-persistent `QualitySettings` value with bounded ceilings and clarity/balanced/fluid preference. Sender readback and limitation warnings remain manual diagnostics. ADR-0007's automatic per-path `HIGH`/`FALLBACK` and unique on-demand shared `LOW` are accepted mainline behavior, but the C report, state controller, and `LOW` runtime are unimplemented. - Chrome 151 synthetic 720p30 propagated balanced/clarity to one host and three viewers without changing peer identities; all viewers kept decoding and fanout stayed host two/relay one. This proves control continuity only. -- The first local A+B alignment foundation binds host capture settings to one uniquely matched outbound track in the same tick, follows `remoteId` and the RTP-specific transport/candidate pair, and blocks stale generations across source replacement. Adjacent `getStats()` deltas rebase on first, empty, ambiguous, changed-identity, or reset intervals. Codec evidence remains MIME-only; this is not the complete A+B contract. +- Local host A+B binds capture settings to one uniquely matched outbound RTP stream in the same tick, follows `remoteId` and the RTP-specific transport/candidate pair, and blocks stale generations across source replacement. Adjacent `getStats()` deltas rebase on first, empty, ambiguous, changed-identity, or reset intervals. The same RTP's transport-bound codec supplies only nullable MIME, self-describing profile token, allowlisted format parameters, and current stream `scalabilityMode`; this diagnostic adds no upload or persistence of raw fmtp/SDP/stats, defaults are not inferred, profile tokens imply no quality/hardware/support conclusion, and multiple RTP/encodings remain unknown. Authenticated C, automatic control, and `LOW` remain unimplemented. - Release `769de201f7cc` passed CI/hygiene/typecheck, 22 files/274 tests, both builds, zero production dependency vulnerabilities, atomic activation, public health/access, exact bundle, and service/log checks. ## Provisional Quality Targets diff --git a/docs/research/realtime-quality-adaptation.md b/docs/research/realtime-quality-adaptation.md index 9d45dc7..c62854d 100644 --- a/docs/research/realtime-quality-adaptation.md +++ b/docs/research/realtime-quality-adaptation.md @@ -131,14 +131,32 @@ device/network identifiers. Opaque server-issued path and connection-generation IDs provide authorization and correlation; a general remote stats stream or telemetry pipeline is unnecessary. -The repository now implements only the first local A+B alignment foundation: -same-tick capture settings plus one uniquely matched outbound RTP sample, -explicit sample/media identity and adjacent deltas, `remoteId` linkage, and the -selected path reached through that RTP stream's transport. Source replacement -blocks sampling and invalidates in-flight generations. This is not the complete -A+B contract: codec evidence is still MIME-only, derived codec -profile/parameters and applicable `scalabilityMode` are absent, and there is no -authenticated C report, two-state controller, or on-demand `LOW` runtime. +The repository now implements the local host A+B foundation: same-tick capture +settings plus one uniquely matched outbound RTP sample, explicit sample/media +identity and adjacent deltas, `remoteId` linkage, and the selected path reached +through that RTP stream's transport. Source replacement blocks sampling and +invalidates in-flight generations. + +Codec evidence follows only that outbound RTP object's `codecId`, and the +referenced `RTCCodecStats` must use the same transport. The local diagnostics +expose nullable, bounded MIME plus a normalized profile token and an ASCII allowlist of +format parameters: H.264 `profile-level-id`, `packetization-mode`, and +`level-asymmetry-allowed`; VP9 `profile-id`, `max-fr`, and `max-fs`; VP8 +`max-fr` and `max-fs`; and AV1 `profile`, `level-idx`, and `tier`. Unknown, +invalid, duplicated, oversized, or non-allowlisted fields such as H.264 +parameter sets are discarded. This diagnostic path does not expose or persist +raw `sdpFmtpLine`, SDP, candidates, or complete stats, and it does not infer +specification defaults. Profile tokens remain self-describing, for example +H.264 `profile-level-id=42e01f`; they are not translated into quality, +hardware, support, or capability conclusions. + +The same unique outbound object supplies nullable current configured +`scalabilityMode`. Sender parameter readback exposes an applied mode only for +one unambiguous encoding. Current quality settings do not request a mode, so +the requested value remains null and a browser-reported default is not called +a mismatch; multiple encodings remain unknown. This still does not implement +an authenticated C report, two-state controller, or on-demand `LOW` runtime, +and browser support remains subject to the controlled matrix. Official W3C text checked 2026-08-19 defines names ending in `Id` as stats-object references. In particular, outbound [`mediaSourceId`](https://www.w3.org/TR/webrtc-stats/#dom-rtcoutboundrtpstreamstats-mediasourceid) @@ -151,6 +169,18 @@ references its selected pair. Therefore an ambiguous RTP set or a broken or cross-transport reference remains unknown; an opaque stats ID is not a safe substitute for those relationships. +The same specification defines [`codecId`](https://www.w3.org/TR/webrtc-stats/#dom-rtcrtpstreamstats-codecid) +as the RTP stream's reference to `RTCCodecStats`, whose `sdpFmtpLine` contains +format-specific negotiated parameters, and defines outbound +[`scalabilityMode`](https://www.w3.org/TR/webrtc-stats/#dom-rtcoutboundrtpstreamstats-scalabilitymode) +as present only when a mode is currently configured for that stream. +WebRTC-SVC separately permits post-negotiation `getParameters()` to return a +different configured mode from one explicitly requested. If no mode was +provided, its absence must not be replaced with an implementation-dependent +default. Chromium currently maps its configured per-stream mode into the +outbound stats field; other browser implementations may omit any optional +member, which remains null rather than a capability conclusion. + ## Accepted Adaptation Direction ADR-0007 rejects room-wide worst-link adaptation. Healthy operation has one @@ -330,6 +360,11 @@ recovery, and that each supported sender cohort can start it reliably. - [W3C WebRTC SVC](https://www.w3.org/TR/webrtc-svc/) - [W3C Media Capabilities](https://www.w3.org/TR/media-capabilities/) - [W3C WebCodecs](https://www.w3.org/TR/webcodecs/) +- [RFC 6184 H.264 RTP payload format](https://www.rfc-editor.org/rfc/rfc6184.html) +- [RFC 7741 VP8 RTP payload format](https://www.rfc-editor.org/rfc/rfc7741.html) +- [RFC 9628 VP9 RTP payload format](https://www.rfc-editor.org/rfc/rfc9628.html) +- [IANA AV1 media type and format parameters](https://www.iana.org/assignments/media-types/video/AV1) +- [Chromium/libwebrtc video stats origins](https://webrtc.googlesource.com/src/+/HEAD/video/g3doc/stats.md) - [MDN `RTCRtpSender.setParameters()`](https://developer.mozilla.org/en-US/docs/Web/API/RTCRtpSender/setParameters) - [Chromium `motion` to libwebrtc `kFluid` bridge](https://chromium.googlesource.com/chromium/src/third_party/+/refs/heads/main/blink/renderer/modules/peerconnection/media_stream_video_webrtc_sink.cc) - [libwebrtc `motion`/`kFluid` sender classification](https://webrtc.googlesource.com/src/+/3b1eab8a69cb5078befb021c5492d3f204a7d6a2/pc/rtp_sender.cc) diff --git a/docs/status.md b/docs/status.md index 5922903..8080354 100644 --- a/docs/status.md +++ b/docs/status.md @@ -17,7 +17,7 @@ unconfigured, so production remains one host connection per viewer and P2P/TURN. - Hidden routing is `direct P2P -> peer-assisted -> optional SFU roots`; TURN is selected per edge, and SFU roots retain bounded peer descendants. Exact-room canaries use `PEER_ASSISTED_ROOM_IDS`; production is off. - Configured fallback adds a non-secret standby URL and token-free DNS/TLS warmup; absent configuration adds no field, import, request, participant, or edge. - Each viewer advertises relay capacity 0/1; detected mobile/iPad clients are leaves. Withdrawal does not move a healthy edge. Browser relays stay one-child and host two-child. -- Production has one memory-only video setting with manual ceilings/readback. Automatic per-path `HIGH`/`FALLBACK` and one on-demand shared `LOW` are accepted but not deployed; only the first local A+B foundation exists. Browser audio is request/presence-only. +- Production has memory-only video settings and manual readback. Automatic `HIGH`/`FALLBACK` plus one shared on-demand `LOW` is accepted but not deployed; host A+B exists, while C/actions do not. Browser audio is request/presence-only. ## Verified Evidence @@ -26,7 +26,7 @@ unconfigured, so production remains one host connection per viewer and P2P/TURN. - Synthetic Chrome 151 720p30 propagated balanced/clarity to one host and three viewers without peer changes; decoding and 2/1 fanout continued. This proves control continuity only. - Chrome 151/LiveKit localhost A/B measured cold active/render at 1.481/2.257 seconds versus standby 0.200/0.320 with host edges at two. It includes SDK/prewarm and is not public-network evidence. - Atomic activation passed health/access, exact bundle, SQLite preservation, services, and logs; the old release is rollback-ready. Responsive 320/375/390 px checks passed without live media. -- The local A+B alignment foundation samples capture settings with one uniquely matched host outbound track, explicit window/media identity and valid adjacent deltas, `remoteId`, and the RTP-bound selected path. Source replacement blocks sampling and discards stale in-flight generations; ambiguous, changed, and reset evidence rebases to unknown. +- Host A+B aligns capture with one unique outbound RTP, interval/identity/deltas, `remoteId`, and path. Its same-transport codec exposes nullable MIME/self-describing profile token/allowlisted fmtp/`scalabilityMode`; tokens imply no quality or capability conclusion, and no raw fmtp/SDP/stats are added. Stale, ambiguous, reset, or cross-transport evidence stays unknown. Independent review found no P1/P2 issues; the final local check passed 22 files/299 tests and both builds. - HTTPS/WSS, access and room/WebSocket auth, renewal, public STUN, and authenticated TURN/UDP/TCP relay-only paths pass; TURN/TLS is off. - Draft native ladder #16/#18/#22/#23/#25/#28 passed one bounded 720p30 two-leg loop with an experiment-only minimum GCC target and one recovered loss; it is not product behavior. @@ -37,7 +37,7 @@ unconfigured, so production remains one host connection per viewer and P2P/TURN. - Silent partitions can wait 30 to 60 seconds for heartbeat detection before the default 5-second grace; this remains unverified. - Real audio, heterogeneous clients, mobile lifecycle, quality/pause, room `1` republish, and sustained profiles are unverified. Poor movie/video audio is user-reported but unclassified; after video A+B, diagnose capture settings, codec/fmtp, actual bitrate, loss, jitter, concealment, jitter buffer, and A/V sync. - Production `769de201f7cc` reportedly sustains native `qualityLimitationReason=bandwidth` and severe blur with one capable LAN/direct viewer; only Host refresh restores quality. Within the five-second grace, stable-clientId Viewer refresh retains its `peerId`/`HostPeer`, while Host refresh rebuilds peers/capture; churn persistence is also reported. This prioritizes but does not prove sender/PC/GCC/capture generation over network or lifecycle causes. -- ADR-0007 is incomplete. Local A+B alignment exists, but codec evidence is MIME-only; derived profile/parameters/`scalabilityMode`, authenticated C, the controller, and `LOW` runtime are absent. Weak paths must ultimately share one `LOW`; fail closed protects `HIGH` only exceptionally, and an unreliable supported cohort fails acceptance. +- ADR-0007 remains incomplete. Host A+B codec fields exist, but browser support and the reported case are unverified; authenticated C, the controller, and `LOW` are absent. Weak paths must share one `LOW`; fail closed protects `HIGH` only exceptionally, and an unreliable supported cohort fails acceptance. - Browser relays re-encode. The native ladder proves one WebCodecs object, not hardware encode. GCC+RTX is no-go; no-RTX weakens stats. Broader audio/loss/routes/reconnect/browser/product/striping gates remain. - The endpoint budget remains host/relay at most two downstream edges; the browser path is host two/viewer one. - ADR-0005 permits SFU only after peer failure. Stable dual-TURN roots are shadow-only; exact-room TURN-root/SFU-root cost, recovery, and trust evidence is absent. @@ -46,9 +46,9 @@ unconfigured, so production remains one host connection per viewer and P2P/TURN. ## Next Milestone -Finish video A+B codec derivation, then reproduce the same-LAN/direct case. -First rebuild one affected `HostPeer`/`connectionId` while retaining capture and -healthy peers; separately compare capture-only and full Host rebuilds with +Use the completed local host A+B diagnostics to reproduce the same-LAN/direct +case. First rebuild one affected `HostPeer`/`connectionId` while retaining +capture and healthy peers; separately compare capture-only and full Host rebuilds with A+B/C evidence and opaque generations. The affected viewer must recover without moving healthy peers or breaking the host-edge cap. Only proven stuck generations justify guarded/cooldown recovery; never reconnect periodically or diff --git a/src/client/components/StatsGrid.tsx b/src/client/components/StatsGrid.tsx index 28e5d37..775ae0d 100644 --- a/src/client/components/StatsGrid.tsx +++ b/src/client/components/StatsGrid.tsx @@ -96,8 +96,17 @@ export function StatsGrid({ title="本地 / 远端候选类型" /> + {metrics.codecProfile && ( + + )} + {metrics.codecParameters && ( + + )} {direction === "send" ? ( <> + {metrics.scalabilityMode && ( + + )} + {(senderParameters.requested.scalabilityMode !== null || + senderParameters.applied.scalabilityMode !== null) && ( + + )} )} diff --git a/src/client/media/quality.ts b/src/client/media/quality.ts index 14e0533..2a366ba 100644 --- a/src/client/media/quality.ts +++ b/src/client/media/quality.ts @@ -57,6 +57,7 @@ export interface VideoSenderParameterValues { maxFramerate: number | null; scaleResolutionDownBy: number | null; degradationPreference: RTCDegradationPreference | null; + scalabilityMode: string | null; } export interface VideoSenderParameterReadback { @@ -170,21 +171,37 @@ function requestedScaleResolutionDownBy( function readVideoSenderParameters( parameters: RTCRtpSendParameters, + includeAppliedScalabilityMode: boolean, ): VideoSenderParameterValues { const encoding = parameters.encodings[0]; + const scalabilityMode = ( + encoding as + | (RTCRtpEncodingParameters & { scalabilityMode?: unknown }) + | undefined + )?.scalabilityMode; return { maxBitrate: encoding?.maxBitrate ?? null, maxFramerate: encoding?.maxFramerate ?? null, scaleResolutionDownBy: encoding?.scaleResolutionDownBy ?? null, degradationPreference: parameters.degradationPreference ?? null, + scalabilityMode: + includeAppliedScalabilityMode && + parameters.encodings.length === 1 && + typeof scalabilityMode === "string" && + /^[A-Za-z0-9_-]{1,32}$/.test(scalabilityMode) + ? scalabilityMode + : null, }; } function sameParameter( key: keyof VideoSenderParameterValues, - requested: number | RTCDegradationPreference | null, - applied: number | RTCDegradationPreference | null, + requested: number | string | null, + applied: number | string | null, ): boolean { + if (key === "scalabilityMode" && requested === null) { + return true; + } if (typeof requested === "number" && typeof applied === "number") { return key === "scaleResolutionDownBy" ? Math.abs(requested - applied) < 0.01 @@ -207,9 +224,9 @@ export async function configureVideoSender( requestedScaleResolutionDownBy(sender, profile); parameters.degradationPreference = profile.degradationPreference; - const requested = readVideoSenderParameters(parameters); + const requested = readVideoSenderParameters(parameters, false); await sender.setParameters(parameters); - const applied = readVideoSenderParameters(sender.getParameters()); + const applied = readVideoSenderParameters(sender.getParameters(), true); const mismatches = ( Object.keys(requested) as Array ).filter((key) => !sameParameter(key, requested[key], applied[key])); @@ -222,6 +239,7 @@ const PARAMETER_LABELS = { maxFramerate: "帧率上限", scaleResolutionDownBy: "分辨率缩放", degradationPreference: "质量优先级", + scalabilityMode: "伸缩模式", } as const satisfies Record; export function senderParameterWarning( diff --git a/src/client/types.ts b/src/client/types.ts index b687bed..5ac4efa 100644 --- a/src/client/types.ts +++ b/src/client/types.ts @@ -36,6 +36,9 @@ export interface ConnectionMetrics { intervalRetransmittedPackets: number | null; intervalRetransmittedBytes: number | null; codec: string | null; + codecProfile: string | null; + codecParameters: string | null; + scalabilityMode: string | null; encoderImplementation: string | null; powerEfficientEncoder: boolean | null; intervalEncodeMs: number | null; @@ -84,6 +87,9 @@ export const EMPTY_METRICS: ConnectionMetrics = { intervalRetransmittedPackets: null, intervalRetransmittedBytes: null, codec: null, + codecProfile: null, + codecParameters: null, + scalabilityMode: null, encoderImplementation: null, powerEfficientEncoder: null, intervalEncodeMs: null, diff --git a/src/client/webrtc/stats.ts b/src/client/webrtc/stats.ts index fc77ace..475540e 100644 --- a/src/client/webrtc/stats.ts +++ b/src/client/webrtc/stats.ts @@ -200,6 +200,135 @@ function linkedRemoteInbound( : null; } +interface CodecEvidence { + codec: string | null; + profile: string | null; + parameters: string | null; +} + +type CodecParameterRule = { + profileKey: string | null; + parameterKeys: readonly string[]; + validators: Readonly>; +}; + +const CODEC_PARAMETER_RULES: Readonly> = { + "video/h264": { + profileKey: "profile-level-id", + parameterKeys: ["packetization-mode", "level-asymmetry-allowed"], + validators: { + "profile-level-id": /^[0-9a-f]{6}$/i, + "packetization-mode": /^[0-2]$/, + "level-asymmetry-allowed": /^[01]$/, + }, + }, + "video/vp9": { + profileKey: "profile-id", + parameterKeys: ["max-fr", "max-fs"], + validators: { + "profile-id": /^[0-3]$/, + "max-fr": /^[1-9][0-9]{0,9}$/, + "max-fs": /^[1-9][0-9]{0,9}$/, + }, + }, + "video/vp8": { + profileKey: null, + parameterKeys: ["max-fr", "max-fs"], + validators: { + "max-fr": /^[1-9][0-9]{0,9}$/, + "max-fs": /^[1-9][0-9]{0,9}$/, + }, + }, + "video/av1": { + profileKey: "profile", + parameterKeys: ["level-idx", "tier"], + validators: { + profile: /^[0-2]$/, + "level-idx": /^(?:[0-9]|[12][0-9]|3[01])$/, + tier: /^[01]$/, + }, + }, +}; + +function linkedVideoCodec( + report: RTCStatsReport, + media: StatsRecord | null, + transport: StatsRecord | null, +): StatsRecord | null { + const codec = getRecord(report, stringValue(media, "codecId")); + const mimeType = stringValue(codec, "mimeType"); + return codec?.type === "codec" && + transport !== null && + stringValue(codec, "transportId") === transport.id && + mimeType !== null && + /^video\/[A-Za-z0-9.+-]{1,32}$/i.test(mimeType) + ? codec + : null; +} + +function deriveCodecEvidence(codec: StatsRecord | null): CodecEvidence { + const mimeType = stringValue(codec, "mimeType"); + if (!mimeType) { + return { codec: null, profile: null, parameters: null }; + } + const rule = CODEC_PARAMETER_RULES[mimeType.toLowerCase()]; + const fmtp = stringValue(codec, "sdpFmtpLine"); + if (!rule || !fmtp || fmtp.length > 2_048) { + return { codec: mimeType, profile: null, parameters: null }; + } + + const segments = fmtp.split(";"); + if (segments.length > 32) { + return { codec: mimeType, profile: null, parameters: null }; + } + const values = new Map(); + const seenKeys = new Set(); + for (const segment of segments) { + const separator = segment.indexOf("="); + if (separator < 1) { + continue; + } + const key = segment.slice(0, separator).trim().toLowerCase(); + const validator = rule.validators[key]; + if (!validator) { + continue; + } + if (seenKeys.has(key)) { + values.delete(key); + continue; + } + seenKeys.add(key); + const value = segment.slice(separator + 1).trim().toLowerCase(); + if (!validator.test(value)) { + continue; + } + values.set(key, value); + } + + const profileValue = rule.profileKey + ? values.get(rule.profileKey) + : undefined; + const parameters = rule.parameterKeys + .flatMap((key) => { + const value = values.get(key); + return value ? [`${key}=${value}`] : []; + }) + .join("; "); + return { + codec: mimeType, + profile: + rule.profileKey && profileValue + ? `${rule.profileKey}=${profileValue}` + : null, + parameters: parameters || null, + }; +} + +function scalabilityModeValue(media: StatsRecord | null): string | null { + const value = stringValue(media, "scalabilityMode"); + return value && /^[A-Za-z0-9_-]{1,32}$/.test(value) ? value : null; +} + export async function collectConnectionMetrics( connection: RTCPeerConnection, direction: "send" | "receive", @@ -351,7 +480,15 @@ export async function collectConnectionMetrics( previous.previousRetransmittedPackets = retransmittedPackets; previous.previousRetransmittedBytes = retransmittedBytes; - const codec = getRecord(report, stringValue(media, "codecId")); + const linkedCodec = linkedVideoCodec(report, media, transport); + const codecEvidence = + direction === "send" + ? deriveCodecEvidence(linkedCodec) + : { + codec: stringValue(linkedCodec, "mimeType"), + profile: null, + parameters: null, + }; const width = numberValue(media, "frameWidth"); const height = numberValue(media, "frameHeight"); const iceProtocol = @@ -401,7 +538,11 @@ export async function collectConnectionMetrics( freezeDurationDelta === null ? null : freezeDurationDelta * 1_000, intervalRetransmittedPackets, intervalRetransmittedBytes, - codec: stringValue(codec, "mimeType"), + codec: codecEvidence.codec, + codecProfile: codecEvidence.profile, + codecParameters: codecEvidence.parameters, + scalabilityMode: + direction === "send" ? scalabilityModeValue(media) : null, encoderImplementation: stringValue(media, "encoderImplementation"), powerEfficientEncoder: booleanValue(media, "powerEfficientEncoder"), intervalEncodeMs, diff --git a/tests/client-state.test.ts b/tests/client-state.test.ts index d8b2857..f5333cc 100644 --- a/tests/client-state.test.ts +++ b/tests/client-state.test.ts @@ -316,6 +316,8 @@ describe("WebRTC stats parsing", () => { transportId: "transport-a", mediaSourceId: "source-a", remoteId: "remote-inbound-a", + codecId: "codec-a", + scalabilityMode: "L2T3_KEY", bytesSent: 2_000, framesEncoded: 60, }), @@ -325,6 +327,7 @@ describe("WebRTC stats parsing", () => { transportId: "transport-z", mediaSourceId: "source-z", remoteId: "remote-inbound-z", + codecId: "codec-z", bytesSent: 99_000, framesEncoded: 99, }), @@ -346,6 +349,19 @@ describe("WebRTC stats parsing", () => { packetsLost: 999, jitter: 0.9, }), + entry("codec-a", "codec", { + transportId: "transport-a", + mimeType: "video/H264", + sdpFmtpLine: + "profile-level-id=42E01F; packetization-mode=1; " + + "level-asymmetry-allowed=1; sprop-parameter-sets=do-not-expose; " + + "x-google-start-bitrate=99999", + }), + entry("codec-z", "codec", { + transportId: "transport-z", + mimeType: "video/VP9", + sdpFmtpLine: "profile-id=3; max-fr=15; max-fs=1200", + }), ]); const connection = { getStats: async () => report as unknown as RTCStatsReport, @@ -372,7 +388,14 @@ describe("WebRTC stats parsing", () => { packetsLost: 2, jitterMs: 4, rttMs: 20, + codec: "video/H264", + codecProfile: "profile-level-id=42e01f", + codecParameters: + "packetization-mode=1; level-asymmetry-allowed=1", + scalabilityMode: "L2T3_KEY", }); + expect(JSON.stringify(metrics)).not.toContain("sprop-parameter-sets"); + expect(JSON.stringify(metrics)).not.toContain("x-google-start-bitrate"); report.get("transport-a")!.selectedCandidatePairId = "pair-z"; const wrongTransportPair = await collectConnectionMetrics( @@ -388,6 +411,21 @@ describe("WebRTC stats parsing", () => { }); report.get("transport-a")!.selectedCandidatePairId = "pair-a"; + report.get("codec-a")!.transportId = "transport-z"; + const wrongTransportCodec = await collectConnectionMetrics( + connection, + "send", + accumulator, + { trackIdentifier: "capture-track-a" }, + ); + expect(wrongTransportCodec).toMatchObject({ + rtpStatsId: "outbound-a", + codec: null, + codecProfile: null, + codecParameters: null, + }); + report.get("codec-a")!.transportId = "transport-a"; + report.get("local-a")!.type = "remote-candidate"; report.get("remote-candidate-a")!.type = "local-candidate"; const malformedCandidateReferences = await collectConnectionMetrics( @@ -416,6 +454,9 @@ describe("WebRTC stats parsing", () => { rtpStatsId: "outbound-a", selectedCandidatePairId: null, path: "unknown", + codec: null, + codecProfile: null, + codecParameters: null, }); report.get("outbound-a")!.transportId = "transport-a"; @@ -440,9 +481,114 @@ describe("WebRTC stats parsing", () => { rtpStatsId: null, selectedCandidatePairId: null, packetsLost: null, + codec: null, + codecProfile: null, + codecParameters: null, + scalabilityMode: null, }); }); + it.each([ + { + mimeType: "video/VP8", + fmtp: "max-fr=60; max-fs=3600", + profile: null, + parameters: "max-fr=60; max-fs=3600", + }, + { + mimeType: "video/VP9", + fmtp: "profile-id=2; max-fr=60; max-fs=3600", + profile: "profile-id=2", + parameters: "max-fr=60; max-fs=3600", + }, + { + mimeType: "video/AV1", + fmtp: "profile=1; level-idx=8; tier=0", + profile: "profile=1", + parameters: "level-idx=8; tier=0", + }, + { + mimeType: "video/H264", + fmtp: + "profile-level-id=42e01f; profile-level-id=invalid; " + + "packetization-mode=1", + profile: null, + parameters: "packetization-mode=1", + }, + { + mimeType: "video/H264", + fmtp: `profile-level-id=42e01f; ignored=${"x".repeat(2_048)}`, + profile: null, + parameters: null, + }, + { + mimeType: "video/H264", + fmtp: [ + "profile-level-id=42e01f", + ...Array.from({ length: 32 }, (_, index) => `ignored-${index}=1`), + ].join(";"), + profile: null, + parameters: null, + }, + { + mimeType: "video/H265", + fmtp: "profile-id=1; tier-flag=0; level-id=93", + profile: null, + parameters: null, + }, + ])( + "derives only allowlisted $mimeType format parameters", + async ({ mimeType, fmtp, profile, parameters }) => { + const report = new Map([ + [ + "transport", + { id: "transport", type: "transport", timestamp: 1_000 }, + ], + [ + "outbound", + { + id: "outbound", + type: "outbound-rtp", + timestamp: 1_000, + kind: "video", + ssrc: 101, + transportId: "transport", + codecId: "codec", + bytesSent: 1_000, + framesEncoded: 30, + }, + ], + [ + "codec", + { + id: "codec", + type: "codec", + timestamp: 1_000, + transportId: "transport", + mimeType, + sdpFmtpLine: fmtp, + }, + ], + ]) as unknown as RTCStatsReport; + const connection = { + getStats: async () => report, + } as unknown as RTCPeerConnection; + + await expect( + collectConnectionMetrics( + connection, + "send", + createStatsAccumulator(), + ), + ).resolves.toMatchObject({ + codec: mimeType, + codecProfile: profile, + codecParameters: parameters, + scalabilityMode: null, + }); + }, + ); + it("rebases sender retransmission deltas after counter rollback", async () => { const outbound = ( timestamp: number, diff --git a/tests/host-peer-tracks.test.ts b/tests/host-peer-tracks.test.ts index edc472b..cacd8ae 100644 --- a/tests/host-peer-tracks.test.ts +++ b/tests/host-peer-tracks.test.ts @@ -222,6 +222,7 @@ function sendStatsReport({ id: "codec", type: "codec", timestamp, + transportId: "transport", mimeType: "video/VP8", }, ], diff --git a/tests/quality.test.ts b/tests/quality.test.ts index 0697e60..58a1395 100644 --- a/tests/quality.test.ts +++ b/tests/quality.test.ts @@ -99,12 +99,14 @@ describe("realtime quality controls", () => { maxFramerate: 60, scaleResolutionDownBy: 4 / 3, degradationPreference: "maintain-resolution", + scalabilityMode: null, }, applied: { maxBitrate: 8_000_000, maxFramerate: 60, scaleResolutionDownBy: 4 / 3, degradationPreference: "maintain-resolution", + scalabilityMode: null, }, mismatches: [], }); @@ -117,7 +119,7 @@ describe("realtime quality controls", () => { const after = { encodings: [{ maxBitrate: 8_000_000 }], degradationPreference: "balanced", - } as RTCRtpSendParameters; + } as unknown as RTCRtpSendParameters; const getParameters = vi .fn<() => RTCRtpSendParameters>() .mockReturnValueOnce(before) @@ -143,6 +145,74 @@ describe("realtime quality controls", () => { expect(senderParameterWarning(readback)).toContain("质量优先级"); }); + it("reports an applied default scalability mode without claiming a request", async () => { + const before = { + encodings: [{}], + } as unknown as RTCRtpSendParameters; + const after = { + encodings: [ + { + maxBitrate: 8_000_000, + maxFramerate: 60, + scaleResolutionDownBy: 1, + scalabilityMode: "L1T2", + }, + ], + degradationPreference: "maintain-resolution", + } as unknown as RTCRtpSendParameters; + const sender = { + track: { getSettings: () => ({ width: 1920, height: 1080 }) }, + getParameters: vi + .fn<() => RTCRtpSendParameters>() + .mockReturnValueOnce(before) + .mockReturnValue(after), + setParameters: vi.fn(async () => undefined), + } as unknown as RTCRtpSender; + + const readback = await configureVideoSender( + sender, + QUALITY_PROFILES["1080p60"], + ); + + expect(readback.requested.scalabilityMode).toBeNull(); + expect(readback.applied.scalabilityMode).toBe("L1T2"); + expect(readback.mismatches).not.toContain("scalabilityMode"); + expect(senderParameterWarning(readback)).toBeNull(); + }); + + it("keeps scalability mode unknown when sender encodings are ambiguous", async () => { + const before = { encodings: [{}] } as RTCRtpSendParameters; + const after = { + encodings: [ + { + rid: "low", + maxBitrate: 8_000_000, + maxFramerate: 60, + scaleResolutionDownBy: 1, + scalabilityMode: "L1T1", + }, + { rid: "high", scalabilityMode: "L1T2" }, + ], + degradationPreference: "maintain-resolution", + } as unknown as RTCRtpSendParameters; + const sender = { + track: { getSettings: () => ({ width: 1920, height: 1080 }) }, + getParameters: vi + .fn<() => RTCRtpSendParameters>() + .mockReturnValueOnce(before) + .mockReturnValue(after), + setParameters: vi.fn(async () => undefined), + } as unknown as RTCRtpSender; + + const readback = await configureVideoSender( + sender, + QUALITY_PROFILES["1080p60"], + ); + + expect(readback.applied.scalabilityMode).toBeNull(); + expect(readback.mismatches).not.toContain("scalabilityMode"); + }); + it("pauses only the video track and can resume it", () => { const { stream, videoTrack } = createVideoStream(); diff --git a/tests/sfu-client.test.ts b/tests/sfu-client.test.ts index 8550356..cd2450f 100644 --- a/tests/sfu-client.test.ts +++ b/tests/sfu-client.test.ts @@ -275,12 +275,14 @@ describe("SfuPublisher", () => { maxFramerate: 60, scaleResolutionDownBy: 1, degradationPreference: "maintain-resolution", + scalabilityMode: null, }, applied: { maxBitrate: 8_000_000, maxFramerate: 60, scaleResolutionDownBy: 1, degradationPreference: "maintain-resolution", + scalabilityMode: null, }, mismatches: [], });