From 834579a7fa945b1bb0c4409f1fe642bcd9385ff4 Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 21 Aug 2026 22:47:46 +0800 Subject: [PATCH] fix(media): raise peer Opus bitrate target [skip ci] --- docs/project-memory.md | 2 +- docs/research/browser-screen-audio-quality.md | 86 +++++++++++-------- docs/research/realtime-quality-adaptation.md | 84 +++++++++--------- docs/方案设计.md | 6 +- docs/需求理解.md | 8 +- src/client/webrtc/screen-audio-sdp.ts | 38 +++++--- tests/screen-audio-sdp.test.ts | 25 ++++-- tests/viewer-peer-generation.test.ts | 4 +- 8 files changed, 148 insertions(+), 105 deletions(-) diff --git a/docs/project-memory.md b/docs/project-memory.md index 8c91786..47dfa28 100644 --- a/docs/project-memory.md +++ b/docs/project-memory.md @@ -31,7 +31,7 @@ Last updated: 2026-08-21 - C+B quality reparenting is edge-local and cooldown-bound; no score, timer, or global parent penalty. - Flagship media is UDP; HTTPS/WSS stays TLS/TCP; the old release remains rollback-only. - Treat settings as ceilings and degradation as unclassified. Use correlated Host A+B/Viewer C and one-variable evidence; never force AV1, infer by UA, or create a composite score. -- Screen audio is route-consistent stereo: Viewer `stereo=1`; SFU uses 128 kbps stereo/`forceStereo`, DTX off, RED retained. Other SDP stays browser-generated; congestion/FEC and stream remain. Production proof open; no audio UI; voice separate. Native audio source-only. +- Screen audio: peer answers use `stereo=1;maxaveragebitrate=128000`; SFU uses 128 kbps stereo/`forceStereo`, DTX off, RED retained. Other SDP/FEC stays browser-owned; one stream; production proof open; no audio UI; voice separate; Native source-only. - Presence reports exact active upstream and feeds Host topology; it does not prove media. Status and details stay neutral until local/current media evidence, then label P2P/SFU and actual TURN; append its protocol only from local relay stats. Web Host names stay socket/localStorage-only; Native remains outside the boundary. - Do not add scene detection, dynamic-FPS control, or forced AV1 without negotiation, encode, game, CPU/GPU, and sender evidence. - ADR-0002 grant/public access, the v3 room-password migration, and Web Host display names are deployed. Names/presence remain session-only without account or roster tables. diff --git a/docs/research/browser-screen-audio-quality.md b/docs/research/browser-screen-audio-quality.md index 8838227..776ec48 100644 --- a/docs/research/browser-screen-audio-quality.md +++ b/docs/research/browser-screen-audio-quality.md @@ -2,9 +2,10 @@ Accessed: 2026-08-21 -Status: the route-consistent screen-media default is source-complete; production -still has a 128 kbps ceiling without an explicit P2P or SFU stereo contract, and -audible route evidence remains open. +Status: the current integration base has route-consistent peer/SFU stereo and a +128 kbps sender ceiling, but peer answers still omit `maxaveragebitrate`, leaving +full-band stereo at libwebrtc's 64 kbps initial codec target. This slice adds the +matching 128 kbps receiver maximum; audible production evidence remains open. ## Scope And Decision @@ -23,8 +24,8 @@ plumbing as quality controls: - treat an audio track as optional and warn before publishing when none exists; - preserve returned audio through source changes, picture pause, P2P, relay, and configured SFU routes; -- keep a `128000` bit/s sender ceiling, Opus stereo receive preference and DTX - off consistently across Web peer and SFU routes; +- keep a `128000` bit/s sender ceiling, matching Opus stereo/maximum-average- + bitrate receive parameters, and DTX off across Web peer and SFU routes; - let the browser's existing WebRTC congestion control reduce actual audio and video traffic; do not add an application audio adaptation loop; - expose no channel-count, sample-rate, codec, bitrate, stereo, DTX, RED, or FEC @@ -48,13 +49,15 @@ contract. It must not turn movie or game audio into a voice-processed source. ## Deployed Reports And Echo Boundary -Users report that current production -`6ccb516a47261054f91dfa2fafa408d39ced59fc` still sounds poor for movie/video -screen audio. That exact release has the 128 kbps ceiling described -above but no explicit stereo contract on peer or SFU routes, making mono -negotiation the strongest code-level candidate. It remains a diagnosis, not an -audible-quality measurement; capture scope, actual fmtp/bitrate, loss, jitter -and concealment still need the existing read-only diagnostics. +Users reported that production +`6ccb516a47261054f91dfa2fafa408d39ced59fc` sounded poor for movie/video screen +audio. That exact release had the 128 kbps ceiling described above but no +explicit stereo contract. The current integration base has since added peer/SFU +stereo, but its peer answers still omit `maxaveragebitrate`; inspected libwebrtc +therefore starts full-band stereo at 64 kbps. This is the strongest remaining +code-level candidate, not an audible-quality measurement: capture scope, actual +fmtp/bitrate, loss, jitter and concealment still need the existing read-only +diagnostics. The earlier delayed self-echo report on `769de201f7cc` has a different boundary: a viewer's voice is rendered by a separate voice application on the Host, @@ -95,7 +98,7 @@ constraints added to `getDisplayMedia()`. | Voice-app exclusion | `restrictOwnAudio` concerns audio produced by the document that invoked capture; `suppressLocalAudioPlayback` concerns local playback of a captured browser surface | The app may observe whether a returned track exists, not which OS processes it contains | Excluding Discord, KOOK, WeChat, notifications, or any other independent process from system audio | Web keeps video-only available and warns that system audio may include calls/notifications; it does not claim isolation | | Capture channels | The Screen Capture specification does not list generic `channelCount` as applicable to display audio | `track.getSettings().channelCount` may describe the returned track when the browser supplies it | That the app controlled the value, or that the RTP encoder sends stereo | Observe only in a future diagnostic; absent means unknown | | Capture sample rate | The Screen Capture specification does not list generic `sampleRate` as applicable to display audio | `track.getSettings().sampleRate` may be present in an implementation | The Opus mode, RTP clock semantics, receiver output rate, or end-to-end fidelity | Observe only in a future diagnostic; absent means unknown | -| RTP send bitrate | `RTCRtpSender.setParameters()` can set `encodings[].maxBitrate` for audio | A following `getParameters()` can show the applied ceiling; outbound byte deltas show actual traffic | A target, minimum, audible improvement, or Opus `maxaveragebitrate`; other limits may keep traffic lower | Request a fixed 128 kbps ceiling, read it back, and retain actual bitrate as the acceptance fact | +| RTP send bitrate | `RTCRtpSender.setParameters()` can set `encodings[].maxBitrate` for audio; Opus `maxaveragebitrate` advertises the receiver's maximum | A following `getParameters()` can show the applied sender ceiling; negotiated fmtp shows the receiver maximum; outbound byte deltas show actual traffic | A minimum, audible improvement, or moment-to-moment rate; congestion may keep traffic lower | Pair the 128 kbps sender ceiling with the same Opus receive maximum and retain actual bitrate as the acceptance fact | | Codec | The user agent chooses among negotiated send codecs unless a separately negotiated codec selection is available | `RTCCodecStats.mimeType` and `sdpFmtpLine` identify the codec and negotiated format parameters in use | That Opus was selected before stats exist, or that negotiated preferences describe actual content | Label only observed negotiated data; do not force a codec | | Stereo | No stable sender parameter controls Opus stereo; RFC 7587 defines `stereo` as the receiver's one-way preference | Capture settings and negotiated codec/fmtp data can be inspected separately | Capture channel count and `opus/48000/2` do not prove encoded stereo | Use one bounded Opus answer transform on the declared Chrome/Edge baseline; no user control or inferred badge | | DTX | The current WebRTC `RTCRtpEncodingParameters` dictionary has no `dtx` member; RFC 7587 defaults absent `usedtx` to `0` | No portable standard sender readback proves DTX operation | A deprecated browser field or SDK option is not a cross-browser contract | Keep peer default-off and set LiveKit `dtx: false`; no user control | @@ -162,20 +165,25 @@ behavior. Current libwebrtc source calculates the default full-band Opus bitrate as 32 kbps times the negotiated channel count. It derives two encoded channels -only from `stereo=1`; otherwise it configures one. That matches the narrow -Chrome observation, whose ordinary offer had no stereo fmtp and sent about -32 kbps. It is an implementation-level explanation for the tested default, -not a portable stereo or bitrate contract. +only from `stereo=1`, so stereo without `maxaveragebitrate` starts from a +64 kbps codec bitrate. `RTCRtpSender.maxBitrate=128000` only caps the allocator; +it does not raise that codec configuration. When present, libwebrtc maps the +negotiated `maxaveragebitrate` value into the Opus encoder bitrate config before +normal congestion adaptation. This is implementation evidence for the current +Chrome/Edge baseline, while the RFC still defines the parameter as a receiver +maximum rather than a minimum or continuous bitrate guarantee. -This makes the current poor movie/music report unsurprising. Production and the -pre-slice source set a 128 kbps sender ceiling, but their Web peer negotiation -does not ask the receiving Viewer to accept stereo. The SFU publisher sets the -same ceiling and `dtx: false`, but its options merge with LiveKit 2.22.0 room -defaults, including `forceStereo: false` and `red: true`; an explicit false -overrides LiveKit's track-settings stereo heuristic. The strongest current -code-level cause is therefore a mono-by-default contract, not evidence that -WebRTC congestion control is broken. Capture/source loss, packet loss, jitter -and concealment can still degrade audio and remain separate diagnostic facts. +The poor movie/music report against production `6ccb516` is consistent with its +historical state. That release and its pre-stereo source set a 128 kbps sender +ceiling, but Web peer negotiation did not request stereo; the SFU publisher set +`dtx: false` while inheriting `forceStereo: false` and `red: true`. That +mono-by-default contract is no longer current. The present integration base +already upserts peer `stereo=1` and publishes SFU audio with `forceStereo: true`, +`dtx: false`, and RED retained. Its remaining peer-specific gap is the absent +`maxaveragebitrate`: libwebrtc starts stereo at 64 kbps despite the 128 kbps +sender ceiling. This slice adds the matching receiver maximum. Capture/source +loss, packet loss, jitter and concealment can still degrade audio and remain +separate diagnostic facts. ## Reference Implementations @@ -194,8 +202,9 @@ No code was copied. All repositories were inspected at pinned commits on `forceStereo: false`; its named presets include stereo music at 64 kbps and high-quality stereo music at 128 kbps. Per-track options merge after those defaults. Its stereo-only `??=` fallback disables DTX/RED only if still - undefined, which the pinned room defaults are not. Screener's pre-slice source - overrode DTX but not `forceStereo` or RED, yielding mono, DTX off and RED on. + undefined, which the pinned room defaults are not. Screener's historical + pre-stereo source overrode DTX but not `forceStereo` or RED, yielding mono, + DTX off and RED on. - lib-jitsi-meet, Apache-2.0, commit [`63a04ec`](https://github.com/jitsi/lib-jitsi-meet/tree/63a04ecabd972ea75e877f9ba12086c13cb68210): [`mungeOpus()`](https://github.com/jitsi/lib-jitsi-meet/blob/63a04ecabd972ea75e877f9ba12086c13cb68210/modules/RTC/TPCUtils.ts#L871-L929) @@ -274,7 +283,7 @@ control: with direct dependencies on MIT-licensed `sdp-transform` 2.15.0 and its TypeScript declarations. Select the single non-rejected audio media section, resolve its Opus payload from structured `rtp` entries, and idempotently - upsert only `stereo=1` in that payload's fmtp before both + upsert `stereo=1` plus `maxaveragebitrate=128000` in that payload's fmtp before both `setLocalDescription()` and signaling the same description. Preserve every other media section, codec, fmtp key and session attribute. Empty SDP, no unambiguous active audio/Opus payload, multiple active audio sections, @@ -285,11 +294,11 @@ control: `createAnswer()` remains usable by `setLocalDescription()`. 3. Do not add `sprop-stereo`: RFC 7587 defines it as a sender-likelihood hint, not the receiver preference that permits the remote encoder to send stereo, - and offer/answer parameters are orthogonal. Do not add `usedtx=0` or - `maxaveragebitrate=128000` either: absent `usedtx` already means off, absent - `maxaveragebitrate` already uses the corresponding Opus mode's recommended - maximum, and `RTCRtpSender.maxBitrate` remains Screener's send ceiling. Keep - the browser's existing `useinbandfec` negotiation untouched. + and offer/answer parameters are orthogonal. Do not add `usedtx=0`; absent + `usedtx` already means off. `maxaveragebitrate=128000` is the receiver maximum + matching Screener's sender ceiling and pinned LiveKit preset, not a minimum; + stock congestion control can still lower actual traffic. Keep the browser's + existing `useinbandfec` negotiation untouched. 4. For SFU publication use LiveKit 2.22.0 `AudioPresets.musicHighQualityStereo`, `forceStereo: true` and `dtx: false`. Do not override the pinned room default `red: true`. RED and Opus in-band FEC @@ -320,10 +329,11 @@ fix; the stable stream already deployed is the A/V synchronization baseline. The implementation budget is deliberately bounded to dependency manifest/lockfile entries, one SDP helper, `ViewerPeer` answer wiring, SFU publisher options, focused unit/integration tests and the owning docs. Parser -tests cover CRLF serialization, an existing/missing fmtp line, unknown fmtp -preservation, idempotence, payload-number lookup, rejected/non-audio sections, -and identity-preserving fallback for empty SDP, no/ambiguous Opus and parser -exceptions. Peer tests prove the exact selected answer is both applied and +tests cover CRLF serialization, an existing/missing fmtp line, exact stereo and +bitrate replacement, unknown fmtp preservation, idempotence, payload-number +lookup, rejected/non-audio sections, and identity-preserving fallback for empty +SDP, no/ambiguous Opus, duplicate targets, and parser exceptions. Peer tests +prove the exact selected answer is both applied and signaled on initial/restart/rebuild paths; SFU tests assert the three explicit options and absence of a RED override. One Chrome/Edge left/right fixture checks channel correctness and a P2P/SFU route switch checks the persistent stream; diff --git a/docs/research/realtime-quality-adaptation.md b/docs/research/realtime-quality-adaptation.md index 33ef386..11a7129 100644 --- a/docs/research/realtime-quality-adaptation.md +++ b/docs/research/realtime-quality-adaptation.md @@ -11,14 +11,13 @@ others. `maintain-framerate` may preserve motion by reducing resolution until game UI, maps, subtitles, and text become unreadable; `maintain-resolution` may instead lower frame rate. Neither preference overrides congestion control. -The user previously accepted `balanced` as the recommended-profile and advanced -default. Later field feedback reported default blur and apparently overcorrected -fluid behavior, so the user explicitly superseded that decision: current source -returns those defaults to clarity-first `maintain-resolution`, while `balanced` -and `maintain-framerate` remain explicit choices. Independently, the P2P -answer-time whole-profile workaround rested on confounded evidence and is -removed below. These preferences still leave actual degradation to the browser; -Screener observes readback and stats. +The user accepted `balanced` as the recommended-profile and advanced default. +A previous iteration temporarily returned those defaults to clarity-first after +field feedback, but that overcorrection is superseded: current source uses +`balanced`, while `maintain-resolution` and `maintain-framerate` remain explicit +choices. Independently, the P2P answer-time whole-profile workaround rested on +confounded evidence and is removed below. These preferences still leave actual +degradation to the browser; Screener observes readback and stats. The inspected Chromium/libwebrtc source chain makes a screen-only shortcut especially unsafe to assume for Screener. The JavaScript `motion` hint reaches @@ -150,13 +149,14 @@ A 2026-08-21 report says that selecting fluid preference on an SFU path could retain low received FPS without reducing the visible resolution. This is not proof that the preference was ignored: `maintain-framerate` is a degradation tradeoff rather than an FPS target. It is also not evidence of SFU temporal -downlayering on Screener's current H.264 `q,h` publication. Pinned LiveKit -server 1.13.5 installs a temporal selector for VP8, but its H.264/H.265 path -installs only the simulcast spatial selector. A `HIGH` ceiling may therefore -let current H.264 BWE choose the lower-resolution `q` representation, not a -lower temporal layer at the same resolution. When resolution remains stable, -classify the case in A/B/C order: capture FPS, Host publisher outbound FPS and -limitation, then Viewer inbound FPS and decode. +downlayering: the current codec is left to automatic negotiation unless the Host +explicitly selects one. Pinned LiveKit server 1.13.5 installs a temporal selector +for VP8, but its H.264/H.265 path installs only the simulcast spatial selector. +With H.264 selected, a `HIGH` ceiling may therefore let BWE choose the +lower-resolution `q` representation, not a lower temporal layer at the same +resolution. When resolution remains stable, classify the case in A/B/C order: +capture FPS, Host publisher outbound FPS and limitation, then Viewer inbound FPS +and decode. Pinned LiveKit client 2.22.0 keeps three relevant pieces of state. Its public `LocalVideoTrack.setDegradationPreference()` updates the saved preference used @@ -190,20 +190,23 @@ bundled into the lifecycle repair. RFC 7742 requires WebRTC browsers to implement both VP8 and H.264 Constrained Baseline, while W3C `setCodecPreferences()` lets the application reorder the -browser's negotiated codec set. The smallest Web policy is therefore to place -all advertised H.264 entries first and retain the complete repair and fallback -list. If H.264 capability is absent or the API rejects the preference, leave the -browser default unchanged. LiveKit publication requests H.264 only when the -sender advertises it, falls back to VP8 otherwise, and keeps `backupCodec=false`; -there is no parallel backup-codec publication. +browser's negotiated codec set. The smallest Web policy is automatic by default: +do not override browser/LiveKit negotiation. Advanced settings can explicitly +place every advertised H.264 or VP8 entry first while retaining the complete +repair and fallback list. A missing capability or rejected preference leaves the +browser default unchanged. LiveKit likewise keeps its default codec for +`automatic`, overrides it only for an explicit selection, and retains +`backupCodec=false`; there is no parallel backup-codec publication. -H.264 is the accepted product preference because its mature cross-device hardware -encode/decode paths address the observed VP8 software-path cost and are more -valuable here than VP8's royalty-free implementation model. This does not claim -an intrinsic bitrate win for every implementation or scene; actual outbound -codec/profile and decoded stats remain the result truth. The choice also does not -prove shared encode: separate browser PeerConnections may construct separate -encoders, and the SFU intentionally publishes exactly the existing `q,h` H.264 +H.264 remains an explicit option because mature cross-device hardware +encode/decode paths may avoid an observed VP8 software-path cost. A controlled +Web comparison also found that forcing H.264 could lower downstream FPS with the +current browser configuration, so the product does not force either codec by +default. Neither observation proves an intrinsic bitrate or efficiency win for +every implementation or scene; actual outbound codec/profile, encoder +implementation and decoded stats remain the result truth. A codec choice also +does not prove shared encode: separate browser PeerConnections may construct +separate encoders, and the SFU still publishes only the configured `q,h` representations. Open-source distribution is not itself a patent-license exemption. This Web @@ -212,7 +215,7 @@ ships no H.264 codec binary or new codec dependency, so licensing uncertainty is not a runtime blocker for the preference. Bundling a codec implementation or changing the distribution/service model still requires a separate license review. -This reversible standard preference is gated by focused ordering, fallback and +These reversible standard preferences are gated by focused ordering, fallback and interoperability tests, not by an exhaustive CPU/GPU/game matrix on one ordinary PC. Primary specifications, maintained implementation behavior, representative target-device observations and sanitized production stats drive product choices. @@ -422,8 +425,8 @@ simulcast encoding. `RemoteTrackPublication.setVideoQuality(HIGH)` sets a per-subscriber spatial-quality ceiling. Server 1.13.5 derives requested spatial/temporal maxima from quality, dimensions and FPS, but applies them through codec-specific selectors: VP8 has temporal selection, while H.264/H.265 -simulcast is spatial-only. The current H.264 candidate can adapt each SFU -downtrack between `q,h` spatial representations and recover it independently. +simulcast is spatial-only. When H.264 is selected, the candidate can adapt each +SFU downtrack between `q,h` spatial representations and recover it independently. Server Dynacast takes the maximum quality requested across subscribers and subscriber nodes, then enables every quality at or below that maximum; a `HIGH` root therefore @@ -600,18 +603,19 @@ resolution, frame rate, or bitrate. current sender with `RTCRtpSender.setParameters()`. It does not reopen the source picker or renegotiate healthy peer connections. - The video track keeps `contentHint = "motion"`; recommended profiles and the - advanced initial value use clarity-first `maintain-resolution`, with explicit - `balanced` and `maintain-framerate` choices. None promises an emitted - resolution or rate. + advanced initial value use `balanced`, with explicit `maintain-resolution` + and `maintain-framerate` choices. None promises an emitted resolution or rate. - `maxBitrate` and `maxFramerate` are ceilings. They are neither minimums nor target guarantees, and the project does not use SDP bitrate hacks. - The folded “advanced video” panel accepts only 720p/1080p/1440p, integer - 15-60 fps, 2-12 Mbps, and the three preferences. It exposes no audio quality - controls and is not renamed in the screen-audio slice. Display capture does + 15-60 fps, 2-12 Mbps, the three preferences, and Automatic/H.264/VP8. Codec + selection is locked during a share and applies to the next one. It exposes no + audio quality controls and is not renamed in the screen-audio slice. Display capture does not standardize channel-count or sample-rate control; the portable audio - `maxBitrate` field remains only a 128 kbps ceiling. The one accepted peer - exception is a structured Viewer-answer Opus `stereo=1` receive preference, - paired with pinned LiveKit's explicit high-quality stereo/forceStereo option. + `maxBitrate` field remains only a 128 kbps ceiling. The accepted peer + exception is a structured Viewer-answer Opus + `stereo=1;maxaveragebitrate=128000` receive contract, paired with pinned + LiveKit's explicit high-quality stereo/forceStereo option. DTX stays fixed off, RED retains the pinned SDK default, and FEC remains browser/SDK negotiation, not a control or custom adaptation algorithm. The full boundary is recorded in `docs/research/browser-screen-audio-quality.md`. @@ -628,7 +632,7 @@ resolution, frame rate, or bitrate. black video and silence without closing the room or media connection. The deployed implementation deliberately stops at manual bounded controls. It -adds no composite score, periodic adjustment, codec forcing, SDP bitrate +adds no composite score, periodic adjustment, automatic codec forcing, SDP bitrate manipulation, or scene detector. Three consecutive samples of one non-`none` native `qualityLimitationReason` produce one explanatory warning; a reason change or recovery resets it and never triggers a media action. diff --git a/docs/方案设计.md b/docs/方案设计.md index 1c90a29..e7ea440 100644 --- a/docs/方案设计.md +++ b/docs/方案设计.md @@ -162,7 +162,7 @@ Web 视频端不做视频 SDP munging。`automatic` 不调用 P2P `setCodecPrefe 当前 Windows Native sender 用本地 opaque ID 保存所选窗口的 HWND、PID 与进程创建时间；helper 在使用前重新验证三者绑定，并以 WASAPI process loopback 的 include-target-process-tree 捕获游戏及其子进程。目标进程无渲染流、退出或捕获失败时显示无声/未知，不回退为整机音频，也不上传 PID、路径、窗口标题、设备标识或原始音频。该路径只支持 Windows 11；Windows 10 game-only 音频保持明确不支持。 -同一 helper 的显式 `native-window-h264` source 用 `CreateForWindow` 建立 WGC item，在所选 adapter 的 free-threaded D3D11 frame pool 接收 BGRA surface；D3D11 VideoProcessor 做固定 1280x720/30 aspect-fit NV12 转换，随后进入 adapter-LUID-bound、hardware-only asynchronous Media Foundation H.264 MFT。输出必须是 Annex-B `42c01f`，首次、周期和反馈恢复帧携带 SPS、PPS 与 IDR。PCM、H.264 access unit 和 local-only hardware status 通过一个有界 helper protocol 进入 generation-bound Go session；H.264 直接写现有 shared fanout，UI 只收一份单向预览副本，PCM 保持现有本地 WebCodecs Opus bridge。任一 Native capture/encode 失败都终止该 source，不静默改用软件 MFT、其他 codec、browser capture、monitor capture 或 system mix；Web 端的 H.264-first 协商与 VP8 fallback 是独立标准 WebRTC 路径。一名普通 Chrome Viewer 已 decoded/rendered 203 帧并收到 500 Opus packets；同一 helper PID 与 adapter LUID 的 `VideoEncode` 观测匹配。 +同一 helper 的显式 `native-window-h264` source 用 `CreateForWindow` 建立 WGC item，在所选 adapter 的 free-threaded D3D11 frame pool 接收 BGRA surface；D3D11 VideoProcessor 做固定 1280x720/30 aspect-fit NV12 转换，随后进入 adapter-LUID-bound、hardware-only asynchronous Media Foundation H.264 MFT。输出必须是 Annex-B `42c01f`，首次、周期和反馈恢复帧携带 SPS、PPS 与 IDR。PCM、H.264 access unit 和 local-only hardware status 通过一个有界 helper protocol 进入 generation-bound Go session；H.264 直接写现有 shared fanout，UI 只收一份单向预览副本，PCM 保持现有本地 WebCodecs Opus bridge。任一 Native capture/encode 失败都终止该 source，不静默改用软件 MFT、其他 codec、browser capture、monitor capture 或 system mix；普通 Web Host 的 `automatic` 默认与显式 H.264/VP8 选择是独立标准 WebRTC 路径。一名普通 Chrome Viewer 已 decoded/rendered 203 帧并收到 500 Opus packets；同一 helper PID 与 adapter LUID 的 `VideoEncode` 观测匹配。 直播中切换来源仍由明确的按钮手势重新调用 `getDisplayMedia()`，不改变房间、host generation 或媒体拓扑。选源取消或失败时继续发送旧流；成功后为每个现有 sender 调用 `replaceTrack()`，健康连接不重新协商。初始 offer 预留 send-only audio transceiver，使音频可以在 `null -> track -> null` 间切换。若某个 peer 的替换或回滚失败，只重建该 peer；新流提交后才停止旧流，并以当前 stream 身份保护旧 track 的迟到 `ended` 事件。统计码率累加器在换源后重置，避免把不同 track 的计数器混算。 @@ -172,9 +172,9 @@ Host 本地预览直接绑定现有 capture stream，不创建 viewer、`RTCPeer 部署版 `769de201f7cc` 的模糊、游戏低帧、持续 `bandwidth` 提示和 Host 刷新恢复都只作为同一 release 的用户报告。首轮复现必须在相同机器、Chrome/Edge、驱动、游戏场景和单个有线 direct Viewer 上对照该 release 与最新 `main`，并分别执行只重建受影响 `HostPeer`、只替换 capture、完整 Host 重建和预览开/关；任何一次只改变一个变量。若实际 codec/encoder 不变而 BWE/发送恢复，则下调 codec 根因优先级；只有编码成本异常时才进入受控 codec spike。 -当前 Web “高级视频设置”不暴露音频质量控件。Screen Capture 规范未把 `channelCount`、`sampleRate` 列为 display-audio 可控属性；`track.getSettings()` 中偶尔出现的值只能作为捕获观测。每个 Web Host、浏览器 relay 或 selected-edge 的实际音频 sender 在首次绑定和换源后通过标准 `RTCRtpSender.setParameters()` 请求并读回 128 kbps ceiling；这只是上限，浏览器 WebRTC 拥塞控制仍可降低实际流量，不新增应用层音频降级器。RFC 7587 将 20 ms full-band stereo music 的建议区间列为 64--128 kbps，pinned LiveKit 2.22.0 也把 128 kbps 命名为 `musicHighQualityStereo`；参数由这些外部证据确定，普通 PC synthetic 只验证 setter/互通，不用于标定听感或性能。 +当前 Web “高级视频设置”不暴露音频质量控件。Screen Capture 规范未把 `channelCount`、`sampleRate` 列为 display-audio 可控属性；`track.getSettings()` 中偶尔出现的值只能作为捕获观测。每个 Web Host、浏览器 relay 或 selected-edge 的实际音频 sender 在首次绑定和换源后通过标准 `RTCRtpSender.setParameters()` 请求并读回 128 kbps ceiling；这只是上限，浏览器 WebRTC 拥塞控制仍可降低实际流量，不新增应用层音频降级器。RFC 7587 将 20 ms full-band stereo music 的建议区间列为 64--128 kbps，pinned LiveKit 2.22.0 也把 128 kbps 命名为 `musicHighQualityStereo`。当前 libwebrtc 在未收到 `maxaveragebitrate` 时把 full-band stereo 的初始 codec bitrate 配成 64 kbps，因此 peer answer 同时声明 128 kbps receiver maximum，和 sender ceiling/SFU preset 对齐；普通 PC synthetic 不用于标定参数、听感或性能。 -Web peer 的最小 stereo 契约只改 Viewer 生成的 answer。实现把 MIT `sdp-transform` 2.15.0 与 `@types/sdp-transform` 声明为 root direct dependencies，不依赖 LiveKit 的 transitive install，也不用字符串或正则拼接 SDP。Helper 仅在解析出唯一非 rejected audio media、通过结构化 `rtp` entry 找到唯一 Opus payload 时，才在该 payload 的 fmtp 幂等 upsert `stereo=1`；随后 `ViewerPeer` 把同一个结果交给 `setLocalDescription()` 和 signaling。首次连接、ICE restart、同 parent rebuild、浏览器 relay child 与 selected-edge TURN 都走该边界；TURN 只改变 ICE transport。空 SDP、无或歧义 Opus、多个有效 audio、畸形 fmtp、parse/write 异常都按对象 identity 返回浏览器原 answer，保住仍可继续的 mono/video edge，不发送半修改 SDP。RFC 7587 说明 `stereo` 是可选单向 receiver preference，缺省为 mono preference；RFC 7874 要求 WebRTC endpoint 实现 Opus，而 W3C 只保证未修改的 `createAnswer()` 可用于 `setLocalDescription()`，所以原 answer 是兼容 fallback。`sprop-stereo` 只是 sender likelihood hint，`usedtx` 缺省已是 0，`maxaveragebitrate` 缺省采用对应模式建议最大值，因此本 slice 不增加这三项，也不改浏览器协商的 `useinbandfec`。 +Web peer 的最小 stereo 契约只改 Viewer 生成的 answer。实现把 MIT `sdp-transform` 2.15.0 与 `@types/sdp-transform` 声明为 root direct dependencies，不依赖 LiveKit 的 transitive install，也不用字符串或正则拼接 SDP。Helper 仅在解析出唯一非 rejected audio media、通过结构化 `rtp` entry 找到唯一 Opus payload 时，才在该 payload 的 fmtp 幂等 upsert `stereo=1` 与 `maxaveragebitrate=128000`；随后 `ViewerPeer` 把同一个结果交给 `setLocalDescription()` 和 signaling。首次连接、ICE restart、同 parent rebuild、浏览器 relay child 与 selected-edge TURN 都走该边界；TURN 只改变 ICE transport。空 SDP、无或歧义 Opus、多个有效 audio、重复目标参数、畸形 fmtp、parse/write 异常都按对象 identity 返回浏览器原 answer，保住仍可继续的 mono/video edge，不发送半修改 SDP。RFC 7587 说明这两个值都是可选的单向 receiver parameters；`maxaveragebitrate` 是 maximum 而非最低保证，实际发送仍可随拥塞下降。当前 libwebrtc 把缺省 full-band stereo codec bitrate 配成 64 kbps，而 `RTCRtpSender.maxBitrate=128000` 只限制上界，不能单独把它提高；显式 128 kbps 参数让 peer 与 LiveKit preset 使用同一 codec 配置目标。`sprop-stereo` 只是 sender likelihood hint，`usedtx` 缺省已是 0，本 slice 不增加二者，也不改浏览器协商的 `useinbandfec`。 SFU publisher 使用 pinned SDK 的 `AudioPresets.musicHighQualityStereo`、`forceStereo: true` 与 `dtx: false`。LiveKit 2.22.0 先合并 room publish defaults，其中 `red: true`；本实现不传 `red` 覆盖值，保留既有 RED request。RED 与 Opus in-band FEC 是不同修复机制，保留前者不声称实际启用、替代或等价于后者。只改 SFU 会导致 route switch 声道行为分叉，因此 P2P/browser relay 与 SFU 必须同一 slice 落地。Viewer 继续在同一个持久 `MediaStream`/媒体元素中增删 screen video/audio track；不新建 audio element、Web Audio mix/resample graph、第二 audio representation 或独立音频时钟。当前每条 outbound edge 的 audio RTP payload ceiling 为 128 kbps，cap2 节点最多 256 kbps（不含 RTP/SRTP/UDP/IP overhead），相对 3--8 Mbps 视频档位较小；stereo 仍可能增加编码与网络成本，且浏览器 relay 不保证跨 PeerConnection shared encode。未来 microphone voice 若进入范围，另建独立 source/track 与 AEC、降噪、gain、DTX 设计，不能作用于电影/游戏屏幕音频。详细证据与 Oopz 未知边界见 `docs/research/browser-screen-audio-quality.md`。 diff --git a/docs/需求理解.md b/docs/需求理解.md index 019519d..46c1fec 100644 --- a/docs/需求理解.md +++ b/docs/需求理解.md @@ -27,7 +27,7 @@ - [ ] 端点 fanout 硬上限：任一非服务器端点任何时刻最多维护两条 active downstream media edges，包括加入、断线、换父和恢复过程；上游接收不计入这项上传预算。普通 Web relay 统一声明最多两个 child，不按 UA、设备类型或页面可见性另建容量策略；实际 edge 失败进入同一有界换父路径。第三名及后续观看者由系统自动分配给通过验收的 peer-assisted 拓扑，不能隐藏地回退为 Host 第三条连接。没有可行 peer path 时，SFU 作为主要中央兜底 virtual parent 正常向一至两个必要 roots 送流；媒体到达 root 后其余可行观看者仍优先沿受限 peer descendants 扩散。多个无法挂到健康 root 的极端 viewer 可成为额外 server-fed 例外，但必须受显式中央 egress/admission 上限约束，不能无界整房 fanout。TURN 只作为可选的极端网络 transport 兼容层，不定义房间拓扑。 - [ ] P2P 优先：每条逻辑媒体边的 ICE 必须先尝试直接 UDP 路径，服务器不得默认中转可直连的媒体。一个或两个观看者优先直接连接分享端；更多观看者的 peer-assisted 行为须先通过 ADR-0004 的实验门槛。 - [ ] NAT 与传输兜底：STUN 必须可用，所有 ordinary peer `RTCPeerConnection` 只接收 STUN ICE servers。PC 失败后按一次 ICE restart、一次同 parent rebuild、一个 alternate peer、再 SFU/UDP virtual parent 的顺序恢复，任一步成功即停止；只有该 edge 的 SFU/UDP 也失败后，controller 才可授权一次短期 authenticated selected-edge TURN/UDP rebuild。授权必须绑定当前 room/share generation、viewer/parent session、route revision 与 old/new connection identity，成功、失败、超时或代次变化即消费；同一房间同时 pending 或 active 的 `peer-selected` attempt 总数最多为一，`host-sfu-ingress` 使用独立状态且不占该 last-mile 名额。不得按 participant、房间或普通 peer PC 下发，Native v2 保持 STUN-only。凭据不进入 URL、日志、浏览器持久化或 SQLite。Coturn 只验证 HMAC 与 expiry，应用必须重验 edge 代次并用短 TTL、one-use state 与 quota 限界。媒体路径保持 UDP-only；全部 UDP 路径耗尽后明确失败。HTTPS/WSS 仍独立使用 TLS/TCP。 -- [ ] 质量控制：保留 1080p60（8 Mbps）、1080p30（5 Mbps）、720p30（3 Mbps）三档推荐上限，三档与高级设置初始值统一采用 `maintain-resolution`（清晰优先），`balanced`（平衡）与 `maintain-framerate`（流畅优先）保留为显式选择；分享期间可直接换档或应用有界高级设置，不重新选择来源或重建健康连接。Peer-assisted relay 与已配置的 SFU 必须接收同一个严格校验的房间级权威设置，覆盖当前和以后建立的 sender；普通 P2P 信令结构保持不变。任何设置都只是浏览器上限和偏好，具体降级由浏览器决定；必须读回实际 sender 参数并通过 stats 观测结果、显示参数拒绝、改写和持续受限原因。不做自定义动态 FPS、综合评分或周期性全房间调档。 +- [ ] 质量控制：保留 1080p60（8 Mbps）、1080p30（5 Mbps）、720p30（3 Mbps）三档推荐上限，三档与高级设置初始值统一采用 `balanced`（平衡）与 `automatic`（自动编码），`maintain-resolution`（清晰优先）、`maintain-framerate`（流畅优先）和 H.264/VP8 保留为显式选择；分享期间可直接换档或应用有界高级设置，不重新选择来源或重建健康连接，codec 在直播中锁定并于下一次分享生效。Peer-assisted relay 与已配置的 SFU 必须接收同一个严格校验的房间级权威设置，覆盖当前和以后建立的 sender；普通 P2P 信令结构保持不变。任何设置都只是浏览器上限和偏好，具体降级由浏览器决定；必须读回实际 sender 参数并通过 stats 观测结果、显示参数拒绝、改写和持续受限原因。不做自定义动态 FPS、综合评分或周期性全房间调档。 - [ ] 路径隔离双表示：direct/peer 继续使用每条 PeerConnection 独立的 stock WebRTC GCC，不把任何 viewer 反馈合成全房目标。每条 SFU path 从 `HIGH` ceiling 开始；优先用 pinned LiveKit 2.22.0/1.13.5 的标准两层 screen-share simulcast，在 Dynacast off 下常驻 exactly `HIGH+LOW`，让内建 per-subscriber BWE 独立降层和恢复。若该 gate 通过，Screener 不实现媒体层 selector。首 gate 使用 zero-descendant SFU leaf；root-with-children 的自动 downshift 与有界迁移是 default-on 前独立硬门。只有内建选择不满足门槛时才用标准 subscriber quality 显式限制，常驻成本不合格时再测标准 sender 手动启停。`LOW` 停编只是资源优化；active representation/layer 硬上限为二，禁止按 viewer 增长。必须实测实际接收层、硬件编码证据、游戏 FPS/p1 low、CPU/GPU、区间 encode cost、上传和分层 bytes。当前 Web P2P simulcast 缺少 receiver 选 RID 和跨 PeerConnection shared-encode 契约；当前 Web/LiveKit SVC 也缺少 direct/peer 独立选层与可移植硬件保证，且 pinned screen share 强制三层 temporal-only `L1T3`，二者按现契约静态 no-go。全部标准路径失败后才考虑 custom/native dual encode。没有合格 hardware/power-efficient 路径或额外表示破坏游戏/上传预算时 fail closed：保护健康路径并明确显示弱路径降级或不可用。 - [ ] 路径级质量状态：应用的 `HIGH`/`FALLBACK` 非对称窗口只决定拓扑资格和诊断，不命令正常的媒体层切换，也不计算综合分数。SFU BWE downshift 先进入 `suspect`；多个连续窗口的 sender/viewer 关联证据确认 `FALLBACK` 后，应用不得稳定保留该 parent，并把 downstream capacity 降为 0。计划或显式质量 fallback 仅是内建路径失败后的候选：已有 children 时先在 connection/assignment generation guard 下全部迁走，失败或代次过期则保持原 assignment/quality。Autonomous downshift 无法 packet-level 预先迁移，default-on 前必须验收 descendants 的短暂影响。只有更长的稳定恢复窗口和 cooldown 后才能重新声明 relay capacity。Viewer 请求必须鉴权、限频、去重，并由 sender 的 GCC/Transport-CC、实际码率、RTT、丢包、重传、编码限制、协商 codec/profile/parameters 与 viewer 解码、丢帧、冻结证据交叉验证；不能仅凭自报生效，也不得上传 raw SDP、candidate 地址或原始设备/网络标识。 - [ ] 断线恢复：临时网络变化时按不同层级做一次性恢复，而不是把同一动作机械重复三次。`failed` 边先做一次 ICE restart，再重建一次同 parent 的 `RTCPeerConnection`，随后尝试一个不同的合格 peer parent；只有 peer 路径耗尽才进入 SFU。`connecting`/`checking` 先显示不改变拓扑的慢连接状态，硬超时必须由真实移动网络证据确定，不能为了更快使用服务器而过早放弃最终可成功的 P2P。任一步恢复即停止后续动作；所有异步结果受 session、route revision、assignment/connection generation 约束。active SFU 只允许一次 fresh-token 重试，再失败即有界回到 peer baseline 或明确失败，并向 Host 显示不含 URL、地址、token 或原始错误的失败阶段。 @@ -42,7 +42,7 @@ - [ ] 私密房间双入口：private room 接受现有 `/r/{code}#v={grant}` 合法邀请直接进入，或“站点访问 + 房间号 + 该房间观看密码”。没有 grant 的 `/join` 或 `/r/{code}` 先显示明确标为“站点访问”的整站验证，通过后才显示明确标为“加入房间”的当前房间密码输入；有效 site-access cookie 直接跳过第一步，但两类密码不得复用、代填或互相替代。Host 可设置、轮换或移除房间观看密码。密码接受 1 至 64 个可见 ASCII 字符；异步 scrypt 全局最多执行 2 个、等待 16 个派生。站点口令和房间密码都不得进入 URL/query/storage/log/SQLite；不存在、错误密码、队满和正确密码后的满房都返回通用结果。 - [ ] 统一自动路由：旗舰部署开启 `PEER_ASSISTED_MEDIA` 后，所有正常房间都进入同一个 direct/peer -> SFU -> selected-edge controller；每个房间独立保存 assignment/session/revision/connection generation 和容量状态。合格父节点构成网状候选关系，但实际媒体保持无环且每个 Viewer 只有一个 active upstream；加入、容量变化、掉线和当前边失败触发有界重选，切换重叠必须受现有边预算与当前代媒体证据约束，不做常驻多源分片。`PEER_ASSISTED_ROOM_IDS` 已退役，room `1` 只作为历史 smoke fixture，不能作为功能开关。未启用总开关的开发/轻量部署仍可保留 ordinary P2P；普通 peer 始终 STUN-only。selected-edge TURN 只能复用当前 controller 选中的异常 edge，不能向普通 peer edge 预发 candidates。该能力不增加 UI、用户选择、百分比发布或第二套路由器。 -- [ ] 质量控制扩展：推荐档位仍是默认入口；“高级视频设置”只允许 720p/1080p/1440p 分辨率 ceiling、15 至 60 的整数 FPS ceiling、2 至 12 Mbps 视频码率 ceiling，以及清晰/平衡/流畅三种偏好，初始值采用 `maintain-resolution`（清晰优先），其余偏好保留为显式选择。直播中显式应用后才生效，避免滑杆产生连续控制流量。本期不重命名该区域，也不增加音频 UI：屏幕音频固定使用一个媒体默认，所有实际浏览器 sender 请求并读回 128 kbps ceiling，DTX 关闭，Web peer 与 SFU 都显式允许 Opus stereo；实际码率仍由浏览器拥塞控制和 stats 决定。Peer 只在唯一明确的 active Opus audio 上做结构化、幂等的 Viewer answer `stereo=1` 变换；非目标形态或 parser/writer 异常必须原样保留浏览器 answer，不得阻断可继续的 mono/video edge。不得用宽泛字符串/正则改 SDP，也不得改写浏览器协商的 FEC。SFU 使用 pinned SDK 的 128 kbps high-quality stereo preset、`forceStereo = true`、`dtx = false`，并保留 pinned room default `red = true`。不得暴露声道、采样率、codec、任意码率、stereo、DTX、RED 或 FEC 控件。实际质量持续低于所选上限时必须显示可解释的限制原因；在真实拥塞、编码器和接收端证据完成前，不把任一组合称为“完美平衡”。未来 microphone voice 若进入范围，必须是独立 track/处理链，不能把 AEC/降噪/DTX 套到电影或游戏音频。 +- [ ] 质量控制扩展：推荐档位仍是默认入口；“高级视频设置”只允许 720p/1080p/1440p 分辨率 ceiling、15 至 60 的整数 FPS ceiling、2 至 12 Mbps 视频码率 ceiling、清晰/平衡/流畅三种偏好，以及 `自动 | H.264 | VP8` 编码选择；初始值采用 `balanced` 与 `automatic`，其余偏好和编码保留为显式选择。视频档位在直播中显式应用；codec 直播中锁定并于下一次分享生效，避免滑杆产生连续控制流量。本期不重命名该区域，也不增加音频 UI：屏幕音频固定使用一个媒体默认，所有实际浏览器 sender 请求并读回 128 kbps ceiling，DTX 关闭，Web peer 与 SFU 都显式允许 Opus stereo 并协商 128 kbps maximum average bitrate；实际码率仍由浏览器拥塞控制和 stats 决定。Peer 只在唯一明确的 active Opus audio 上做结构化、幂等的 Viewer answer `stereo=1;maxaveragebitrate=128000` 变换；非目标形态或 parser/writer 异常必须原样保留浏览器 answer，不得阻断可继续的 mono/video edge。不得用宽泛字符串/正则改 SDP，也不得改写浏览器协商的 FEC。SFU 使用 pinned SDK 的 128 kbps high-quality stereo preset、`forceStereo = true`、`dtx = false`，并保留 pinned room default `red = true`。不得暴露声道、采样率、codec、任意码率、stereo、DTX、RED 或 FEC 控件。实际质量持续低于所选上限时必须显示可解释的限制原因；在真实拥塞、编码器和接收端证据完成前，不把任一组合称为“完美平衡”。未来 microphone voice 若进入范围，必须是独立 track/处理链，不能把 AEC/降噪/DTX 套到电影或游戏音频。 - [ ] SFU 质量设置生命周期：直播中修改档位时，浏览器 sender 的 applied readback、LiveKit `LocalVideoTrack` 保存的 degradation preference、track restart options 与 `LocalTrackPublication` 的 republish options 必须作为同一次串行操作更新；任一步失败恢复旧 sender 与旧 SDK 保存态，旧 publication generation 不得回写当前状态。LiveKit full reconnect/republish 不得恢复初次发布的旧偏好。该修复不改变 codec、码率上限、层数或路由，也不能单独解释即时低帧。 - [ ] 本地预览性能候选：在相同动态场景中 A/B 测量 Host 动态预览开/关的 CPU、GPU、encode time 和实际 FPS。只有稳定收益且捕获、编码和远端画面不受影响时，才允许直播开始后默认显示静止缩略图或隐藏动态预览并提供按需展开；未测前不宣称收益。 - [ ] Peer-assisted 媒体路径：分享端与每名普通 Web viewer 都最多两个 child。信令服务维护可从多个合格节点选择 parent 的候选图，但每名 viewer 任一时刻只有一个 active upstream，活动媒体图保持无环；只在新加入、容量释放、节点断开或当前 edge 失败时按深度和服务端加入序号做有界分配或换父，不加入综合评分、主动重平衡、多源媒体或持续优化。每个 viewer 将收到的标准 remote track 通过最多两个新的 `RTCPeerConnection` 转发，因此会逐跳解码和分别重编码；必须测量而不能称为 shared encode。服务端只在受房间数限制的内存状态中保存当前严格校验的质量设置，不写入 SQLite；host 鉴权后先重报设置，在线 viewers 接收更新，后加入者从鉴权快照取得当前值。 @@ -61,7 +61,7 @@ ### P2 - 可选功能 - [ ] 证据驱动的统一 relay capacity：当前上线档固定为 `2`，不是永久产品上限。后续只在独立验收后允许所有 Web relay 使用同一套离散 `0/1/2/3` 容量状态，不按设备或 UA 分类；持续 sender bitrate、`availableOutgoingBitrate`、区间 encode time、`qualityLimitationReason`、loss 与 RTT 健康窗共同决定升降，服务端 deployment hard ceiling 与 cooldown 继续限界。H.264、hardware encode 或 shared encode 可以降低编码成本或同观感码率，但每个 child 仍消耗一份网络副本。本次 slice 不实现动态控制。 -- [x] 原生 Windows 捕获辅助程序：显式 `native-window-h264` 模式使用同一 bound window target 的 WGC/D3D11 画面与 WASAPI process-tree PCM，并用 adapter-bound、hardware-only Media Foundation H.264 `42c01f` 直接进入现有 Go fanout；Web 端独立使用标准 H.264-first 协商并保留 VP8 回退，不允许运行时静默改用软件编码、显示器画面或系统混音。一名普通 Chrome Viewer 已 decoded/rendered 203 帧并收到 500 Opus packets，同一 helper PID/adapter 的 `VideoEncode` 与 LUID 匹配。 +- [x] 原生 Windows 捕获辅助程序：显式 `native-window-h264` 模式使用同一 bound window target 的 WGC/D3D11 画面与 WASAPI process-tree PCM，并用 adapter-bound、hardware-only Media Foundation H.264 `42c01f` 直接进入现有 Go fanout；普通 Web Host 独立采用默认 `automatic`、可显式选择 H.264/VP8 的标准协商路径，不允许运行时静默改用软件编码、显示器画面或系统混音。一名普通 Chrome Viewer 已 decoded/rendered 203 帧并收到 500 Opus packets，同一 helper PID/adapter 的 `VideoEncode` 与 LUID 匹配。 - [ ] Viewer 向电视输出：系统级屏幕镜像现在可直接使用；产品只复用平台 AirPlay/Cast，不自造投屏协议。截至 2026-08-21，Chromium 的 Remote Playback URL 路径和 WebKit 明确拒绝 wireless target 的 live `MediaStream` player 均未给当前 WebRTC `srcObject` 提供可用契约，因此不显示伪可用入口。只在浏览器实现明确支持并由实体 Apple TV 或 Chromecast/Google TV 证明当前 live 画面与音频后，才按通过的平台加入 Viewer 本地图标按钮。投屏不能改变 room、TURN/SFU/peer route、Host fanout 或服务端媒体分发；Google Cast receiver 需要可获取的媒体 URL，当前 P2P `MediaStream` 不满足该输入，因此不加入 Cast SDK/receiver。 - [ ] 多地区 SFU-root 调度，以及可选 TURN 兼容层的按地区、运营商可达率。 @@ -158,7 +158,7 @@ - Given 浏览器未返回系统音轨，When 分享开始，Then 分享者在启播前看到“当前来源没有可共享音频”的提示。 - Given 分享者选择标签页或窗口，When 浏览器支持对应音频捕获，Then picker 提供该来源音频而不扩大为系统混音。Given 分享者选择整个屏幕，When 浏览器支持系统音频，Then picker 提供由用户决定的系统音频开关。应用同时请求 `windowAudio: "window"` 和 `systemAudio: "include"`，不根据 track 存在性伪造来源标签。 - Given 浏览器返回分享音轨，When 开始发布，Then 应用设置标准 `contentHint = "music"` 作为内容意图元数据；该值可读回，但不作为 Opus、码率、声道、采样率、FEC、DTX 或立体声的质量证据。 -- Given Web Host、浏览器 relay 或 selected-edge 建立或替换分享音轨，When 对实际音频 sender 应用发送参数，Then 从当次 `getParameters()` 派生 `encodings[0].maxBitrate = 128000`、调用 `setParameters()` 并立即读回；Given Web Viewer 为含唯一有效 audio/Opus payload 的 offer 生成 answer，Then 用结构化 SDP parser 只 upsert 该 payload 的 `stereo=1`，并把同一个结果用于 `setLocalDescription()` 和 signaling。首次连接、ICE restart、同 parent rebuild 和 relay child 都适用；空 SDP、无/歧义 Opus、多个有效 audio、畸形 fmtp 或 parse/write 异常必须按对象 identity 返回浏览器原 answer 并继续，不能发送部分修改或静默冒充 stereo。只有实际 `setLocalDescription()` 失败才进入既有连接错误路径。`sprop-stereo`、`usedtx` 与 `maxaveragebitrate` 不新增，其他 fmtp/FEC/session/media 字段逐项保留。 +- Given Web Host、浏览器 relay 或 selected-edge 建立或替换分享音轨，When 对实际音频 sender 应用发送参数，Then 从当次 `getParameters()` 派生 `encodings[0].maxBitrate = 128000`、调用 `setParameters()` 并立即读回；Given Web Viewer 为含唯一有效 audio/Opus payload 的 offer 生成 answer，Then 用结构化 SDP parser 只 upsert 该 payload 的 `stereo=1` 与 `maxaveragebitrate=128000`，并把同一个结果用于 `setLocalDescription()` 和 signaling。首次连接、ICE restart、同 parent rebuild 和 relay child 都适用；空 SDP、无/歧义 Opus、多个有效 audio、重复目标参数、畸形 fmtp 或 parse/write 异常必须按对象 identity 返回浏览器原 answer 并继续，不能发送部分修改或静默冒充 stereo/128 kbps。只有实际 `setLocalDescription()` 失败才进入既有连接错误路径。`sprop-stereo` 与 `usedtx` 不新增，其他 fmtp/FEC/session/media 字段逐项保留。 - Given 通过 SFU 发布分享音轨，When 构建 LiveKit 2.22.0 publish options，Then 使用 `AudioPresets.musicHighQualityStereo`、`forceStereo = true` 与 `dtx = false`，不覆盖 pinned room default `red = true`；验收同时核对 SDK 选项、applied ceiling、当前 RTP codec/fmtp 与 stats 实际码率，不能从 preset 名称或 `opus/48000/2` 单独推断实际声道内容。 - Given 浏览器返回屏幕音轨，When 产品展示音频状态或未来连接详情，Then 只能把存在性、浏览器实际返回的 track settings、实际协商 codec/fmtp 和实际流量分别标为捕获、协商或观测事实；字段缺失必须显示为未知，且不得从 Opus 的 `48000/2` RTP 声明推断源采样率或实际 stereo。 - Given Web 或后续 native sender 分享电影/视频音频，When 进行产品验收，Then 必须用独立 A/B/C 证据关联 capture audio track/settings、实际 negotiated codec/fmtp、outbound/inbound 实际码率、loss、jitter、concealment、jitter-buffer 与 A/V sync；P2P、浏览器 relay 与 SFU route switch 保持同一持久 Viewer `MediaStream`/媒体元素和 stereo 协商，不新增独立 audio element 或 Web Audio 时钟边界。native sender 必须通过该音质和同步矩阵，且不得把 audio `maxBitrate` ceiling 当作提质证明。 diff --git a/src/client/webrtc/screen-audio-sdp.ts b/src/client/webrtc/screen-audio-sdp.ts index beb3ab7..852c9d6 100644 --- a/src/client/webrtc/screen-audio-sdp.ts +++ b/src/client/webrtc/screen-audio-sdp.ts @@ -1,5 +1,15 @@ import { parse, parsePayloads, write } from "sdp-transform"; +import { SCREEN_AUDIO_MAX_BITRATE } from "../media/quality"; + +const OPUS_RECEIVE_PREFERENCES = { + stereo: "1", + maxaveragebitrate: String(SCREEN_AUDIO_MAX_BITRATE), +} as const; +const OPUS_RECEIVE_FMTP = Object.entries(OPUS_RECEIVE_PREFERENCES) + .map(([name, value]) => `${name}=${value}`) + .join(";"); + export function preferScreenAudioStereo( answer: RTCSessionDescriptionInit, ): RTCSessionDescriptionInit { @@ -41,13 +51,13 @@ export function preferScreenAudioStereo( return answer; } if (formats[0]) { - const config = upsertStereo(formats[0].config); + const config = upsertOpusReceivePreferences(formats[0].config); if (config === null) { return answer; } formats[0].config = config; } else { - audio.fmtp.push({ payload, config: "stereo=1" }); + audio.fmtp.push({ payload, config: OPUS_RECEIVE_FMTP }); } const sdp = write(session); @@ -57,25 +67,31 @@ export function preferScreenAudioStereo( } } -function upsertStereo(config: string): string | null { +function upsertOpusReceivePreferences(config: string): string | null { const parameters = config.split(";").map((parameter) => parameter.trim()); - let stereoIndex = -1; + const preferenceIndexes = new Map(); for (const [index, parameter] of parameters.entries()) { const equals = parameter.indexOf("="); if (equals <= 0 || equals === parameter.length - 1) { return null; } - if (parameter.slice(0, equals).trim().toLowerCase() === "stereo") { - if (stereoIndex !== -1) { + const name = parameter.slice(0, equals).trim().toLowerCase(); + if (Object.hasOwn(OPUS_RECEIVE_PREFERENCES, name)) { + if (preferenceIndexes.has(name)) { return null; } - stereoIndex = index; + preferenceIndexes.set(name, index); } } - if (stereoIndex === -1) { - parameters.push("stereo=1"); - } else { - parameters[stereoIndex] = "stereo=1"; + + for (const [name, value] of Object.entries(OPUS_RECEIVE_PREFERENCES)) { + const parameter = `${name}=${value}`; + const index = preferenceIndexes.get(name); + if (index === undefined) { + parameters.push(parameter); + } else { + parameters[index] = parameter; + } } return parameters.join(";"); } diff --git a/tests/screen-audio-sdp.test.ts b/tests/screen-audio-sdp.test.ts index fec6061..6c24767 100644 --- a/tests/screen-audio-sdp.test.ts +++ b/tests/screen-audio-sdp.test.ts @@ -25,7 +25,7 @@ function answerSdp(opusFmtp: string | null = "minptime=10;useinbandfec=1"): stri } describe("preferScreenAudioStereo", () => { - it("adds one exact stereo token without changing FEC or video fmtp", () => { + it("adds exact stereo and bitrate tokens without changing FEC or video fmtp", () => { const result = preferScreenAudioStereo({ type: "answer", sdp: answerSdp("minptime=10;useinbandfec=1;x-stereo=keep"), @@ -34,7 +34,8 @@ describe("preferScreenAudioStereo", () => { expect(session.media[0]?.fmtp).toContainEqual({ payload: 111, - config: "minptime=10;useinbandfec=1;x-stereo=keep;stereo=1", + config: + "minptime=10;useinbandfec=1;x-stereo=keep;stereo=1;maxaveragebitrate=128000", }); expect(session.media[1]?.fmtp).toEqual([ { payload: 96, config: "packetization-mode=1;profile-level-id=42e01f" }, @@ -42,14 +43,16 @@ describe("preferScreenAudioStereo", () => { expect(preferScreenAudioStereo(result)).toEqual(result); }); - it("replaces only an exact stereo token", () => { + it("replaces only exact stereo and bitrate tokens", () => { const result = preferScreenAudioStereo({ type: "answer", - sdp: answerSdp("stereo=0;x-stereo=0;minptime=10"), + sdp: answerSdp( + "stereo=0;x-stereo=0;maxaveragebitrate=64000;x-maxaveragebitrate=keep", + ), }); expect(parse(result.sdp!).media[0]?.fmtp[0]?.config).toBe( - "stereo=1;x-stereo=0;minptime=10", + "stereo=1;x-stereo=0;maxaveragebitrate=128000;x-maxaveragebitrate=keep", ); }); @@ -61,7 +64,7 @@ describe("preferScreenAudioStereo", () => { expect(parse(result.sdp!).media[0]?.fmtp).toContainEqual({ payload: 111, - config: "stereo=1", + config: "stereo=1;maxaveragebitrate=128000", }); }); @@ -79,6 +82,7 @@ describe("preferScreenAudioStereo", () => { expect(audio[0]?.fmtp[0]?.config).toBe("stereo=0"); expect(audio[1]?.fmtp[0]?.config).toContain("stereo=1"); + expect(audio[1]?.fmtp[0]?.config).toContain("maxaveragebitrate=128000"); }); it.each([ @@ -104,6 +108,15 @@ describe("preferScreenAudioStereo", () => { ), }, ], + [ + "duplicate target parameter", + { + type: "answer", + sdp: answerSdp( + "stereo=0;maxaveragebitrate=64000;MAXAVERAGEBITRATE=96000", + ), + }, + ], ] as const)("returns the original answer for %s", (_case, answer) => { const result = preferScreenAudioStereo(answer); expect(result).toBe(answer); diff --git a/tests/viewer-peer-generation.test.ts b/tests/viewer-peer-generation.test.ts index c376bbc..4b0c21a 100644 --- a/tests/viewer-peer-generation.test.ts +++ b/tests/viewer-peer-generation.test.ts @@ -207,7 +207,7 @@ afterEach(() => { }); describe("ViewerPeer connection generations", () => { - it("sets and signals stereo for initial, restart, and rebuild answers", async () => { + it("sets and signals stereo audio bitrate for every answer", async () => { const signals: SignalPayload[] = []; const peer = createPeer(signals, []); @@ -224,7 +224,7 @@ describe("ViewerPeer connection generations", () => { expect(description).toEqual(expect.objectContaining({ type: "answer", sdp: expect.stringContaining( - "a=fmtp:111 minptime=10;useinbandfec=1;stereo=1", + "a=fmtp:111 minptime=10;useinbandfec=1;stereo=1;maxaveragebitrate=128000", ), })); expect(answers[index]?.description.sdp).toBe(description.sdp);