From e87498d414888373751d039f2ea86f1e4bb0028f Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 21 Aug 2026 00:44:38 +0800 Subject: [PATCH] feat(native): add opt-in H264 sender path (#85) Adds an explicit H.264 Native sender mode while preserving VP8 as the default, with one bounded Host/Viewer loopback and an explicit hardware-attribution caveat. --- docs/README.md | 1 + docs/project-memory.md | 7 +- .../research/native-h264-hardware-decision.md | 6 ++ docs/research/native-h264-opt-in-path.md | 67 ++++++++++++++++ docs/status.md | 9 ++- native/sender/internal/app/server.go | 19 ++++- native/sender/internal/app/server_test.go | 13 ++++ native/sender/internal/app/ui/app.js | 20 +++-- native/sender/internal/app/ui/index.html | 7 ++ native/sender/internal/media/fanout.go | 76 ++++++++++++++++--- native/sender/internal/media/fanout_test.go | 33 ++++++++ native/sender/internal/remote/codec.go | 56 ++++++++++++++ native/sender/internal/remote/codec_test.go | 66 ++++++++++++++++ native/sender/internal/remote/peer.go | 18 ++++- native/sender/internal/remote/session.go | 51 +++++++++---- scripts/native-one-viewer-gate.ts | 10 ++- tests/native-hardware-status.test.ts | 10 ++- 17 files changed, 421 insertions(+), 48 deletions(-) create mode 100644 docs/research/native-h264-opt-in-path.md create mode 100644 native/sender/internal/remote/codec.go create mode 100644 native/sender/internal/remote/codec_test.go diff --git a/docs/README.md b/docs/README.md index 0d7568c..2aa85e8 100644 --- a/docs/README.md +++ b/docs/README.md @@ -13,6 +13,7 @@ - [Advanced peer distribution](./research/advanced-peer-distribution.md): multi-tree/SVC, encoded-object relay, native RTP forwarding, FEC/network-coding, MoQ, and measurable go/no-go gates. - [Native shared-encode sender](./research/native-shared-encode-sender.md): consolidated Draft #16/#18/#22/#23/#25/#28 ladder, unclassified product-gate failure, staged revalidation, and stop line. - [Native H.264 hardware decision](./research/native-h264-hardware-decision.md): bounded WebCodecs no-go, proved Media Foundation/NVIDIA hardware encode, the default-fmtp `42c01f` stop, and the retained interop boundary. +- [Native H.264 opt-in path](./research/native-h264-opt-in-path.md): explicit H.264 sender mode, exact Pion profile registration, and one bounded Host/Viewer loopback with hardware caveats. - [Realtime quality adaptation](./research/realtime-quality-adaptation.md): correlated capture/send/receive diagnosis, production generation-rebuild gate, path-isolated dual representations, and bounded simulcast/Dynacast/SVC checks. - [Browser screen-audio quality](./research/browser-screen-audio-quality.md): capture/source compatibility, standard readback limits, Opus negotiation boundaries, and the retained no-control decision. - [Display name and presence boundaries](./research/display-name-presence.md): browser-local storage failure behavior, Unicode normalization, control-character policy, and the Web-only presence capability boundary. diff --git a/docs/project-memory.md b/docs/project-memory.md index 1cf76ef..b4fcea0 100644 --- a/docs/project-memory.md +++ b/docs/project-memory.md @@ -1,6 +1,6 @@ # Project Memory -Last updated: 2026-08-20 +Last updated: 2026-08-21 ## Confirmed Intent @@ -23,7 +23,7 @@ Last updated: 2026-08-20 - Use direct host P2P for one or two viewers. Keep ADR-0004 off broadly; use an isolated exact-room candidate until resource, quality, recovery, SFU/UDP, bounded-failure, and browser/mobile gates pass. - Browser relays resend remote `MediaStreamTrack` values and re-encode at each hop; WebRTC does not guarantee a shared encoder across peer connections, so measure the cost. - Keep experiments bounded: the standard representation sequence is below; native RTP relay, encoded-object striping, and FEC remain separate. -- ADR-0006 remains no-go for expansion. A Chrome 151 loopback proves fixed VP8 WebCodecs -> Go RTP -> one 1280x720 Viewer: 30 sender frames/85 source RTP packets, 877 inbound packets, 299 decoded and rendered, no fatal. Pion outbound delta was not retained because its 2-second diagnostics snapshot did not refresh. Hardware, multi-viewer/FIFO, endurance, public-network, and packaged-native proof remain open. Source fixes timestamp overlap from nullable duration; the separate MF RTX fixture still misses Pion fmtp. Native v2 remains memory-only and leaves SQLite/Web unchanged. +- ADR-0006 remains no-go. Native has explicit H.264 opt-in beside default VP8: Chrome 151 loopback reached 299 decoded/298 rendered frames (18 sender, 156 source RTP, 2,690 inbound) at 1280x720 with zero encoder/fatal errors. The gate retains a Pion snapshot-timing residual. `prefer-hardware` is only a request; physical attribution, multi-viewer, endurance, public-network, and packaged-native proof remain open. - Mobile Web Host is unsupported; feature-detect and fail clearly. Mobile Viewer stays leaf-only. After Windows native, gate Android 14+; iOS waits for stable iOS 27 ScreenCaptureKit. - ADR-0005 accepts direct/peer UDP, bounded SFU roots, then optional selected-edge TURN. Source has a default-off, undeployed one-shot rebuild after SFU; participant-wide TURN remains rejected and production stays STUN-only. - Keep the controller exact-room only: room `1` is the STUN/SFU smoke. Preserve sticky progressing P2P, mobile leaves and break-before-make. Recovery spends one attempt per layer (ICE restart, same-parent rebuild, alternate peer, then SFU), never three identical retries; active SFU gets one fresh grant before Peer failback. @@ -50,6 +50,7 @@ Last updated: 2026-08-20 - Room `1` publishes `HIGH+LOW` with Dynacast off and subscriber `HIGH` ceilings, and now has bounded local quality reparenting. Both remain unverified on real media; test zero-child leaves and calibrated C+B loss before broad rollout. - C+B uses three hard-bad pairs, one-use samples and guarded intent/cooldown. Deployed admission rescue moves the oldest childless zero-capacity Host leaf below an unassigned one-slot relay in one revision; no score/timer/global rebalance. Deployment passed, but no real room triggered it. - Source-only Web Host name/presence leaves Native wire/media unchanged and separates Viewer roster from Host diagnostics; the Host-name artifact was not activated. +- Native H.264 opt-in is source-complete for one loopback; VP8 remains default and Web/production are unchanged. See `docs/research/native-h264-opt-in-path.md`. ## Provisional Quality Targets @@ -69,7 +70,7 @@ These are measurement gates, not performance claims. - Project license and distribution model, which determines whether GPL/AGPL sources can move beyond study-only use. - Initial deployment regions and expected mainland China, Hong Kong, and overseas network mix. - Whether voice chat ever enters scope or Screener stays complementary to an existing voice application. -- Production calibration of local-reparent thresholds and the ADR-0007 native matrix; #28's minimum-of-two rule is not a candidate. +- Production calibration of local-reparent thresholds, physical H.264 hardware attribution, and the ADR-0007 native matrix; #28's minimum-of-two rule is not a candidate. ## Source Of Truth diff --git a/docs/research/native-h264-hardware-decision.md b/docs/research/native-h264-hardware-decision.md index 30a7661..6b81e4a 100644 --- a/docs/research/native-h264-hardware-decision.md +++ b/docs/research/native-h264-hardware-decision.md @@ -4,6 +4,12 @@ Date: 2026-08-20 Status: `no-go-native-h264-hardware-pinned-fmtp` +The later source-only functional slice in +[`native-h264-opt-in-path.md`](./native-h264-opt-in-path.md) registers the +fixture profile explicitly and proves one H.264 Host/Viewer loopback. It does +not change this spike's physical-hardware decision: WebCodecs hardware +attribution remains unverified and production remains VP8/default. + This spike does not amend ADR-0006 or authorize another product, room, or production run. It answers one narrow question: can a bounded Windows H.264 path satisfy the project's physical hardware-encode and existing default Pion diff --git a/docs/research/native-h264-opt-in-path.md b/docs/research/native-h264-opt-in-path.md new file mode 100644 index 0000000..7d33135 --- /dev/null +++ b/docs/research/native-h264-opt-in-path.md @@ -0,0 +1,67 @@ +# Native H.264 Opt-In Path + +## 2026-08-21 00:15 + +The native sender now has a small, explicit H.264 mode. VP8 remains the default +and the Web sender is unchanged. A sender chooses H.264 in its local Codec +control; the choice is carried only through the loopback start/media handshake. + +The H.264 path is: + +```text +Chrome WebCodecs H.264 Annex-B + -> existing 17-byte local frame envelope + -> one Go queue and Pion H264Payloader + -> one TrackLocalStaticRTP + -> at most two independent PeerConnections +``` + +The Go session creates a private Pion `MediaEngine` only for H.264 sessions. It +registers the fixture-derived Constrained Baseline `profile-level-id=42c01f`, +`packetization-mode=1`, and a dedicated dynamic payload type, then restricts +the sender transceiver to that H.264 capability. The shared fanout and keyframe +feedback remain unchanged. The default VP8 session still uses Pion's default +API and payloader. + +## Bounded Loopback Evidence + +One short Chrome 151 one-viewer gate ran on 2026-08-20/21 with the in-memory +development server, no peer assistance, no TURN, and a synthetic animated +1280x720 source. The H.264 sender configuration was 720p30, 3 Mbps, one +encoder instance, Annex-B output, and `hardwareAcceleration: prefer-hardware`. + +Retained evidence: + +| Boundary | Result | +| --- | --- | +| Sender configuration / encoder | H.264 config accepted; one encoder; 18 frames written; 156 source RTP packets; 174,978 source RTP bytes; zero encoder/fatal errors | +| Signaling and ICE | Host authentication, room creation, offer/answer, trickle ICE, and both peer connections reached connected | +| Viewer media | 2,690 inbound packets, about 3.0 MiB, 299 decoded and 298 rendered frames, 1280x720 | +| Resource/lifecycle | One host edge; cleanup completed; no production endpoint or TURN allocation | + +The generic gate reported failure only because its two-second Pion diagnostics +snapshot did not refresh during the final sample (`pionPacketDelta=0`), the same +probe-timing residual retained by the earlier VP8 run. All independent viewer +receive/decode/render and sender checks passed. This is one functional +loopback, not a public-network, multi-viewer, endurance, or codec benchmark. + +## Hardware Boundary + +The Media Foundation/NVIDIA fixture remains the physical hardware evidence and +informed the exact `42c01f` profile and in-band SPS/PPS/keyframe contract. The +product sender requests WebCodecs H.264 hardware acceleration, but browser +`isConfigSupported()` and the preference string do not prove which encoder was +selected. The UI therefore reports hardware preference separately from hardware +evidence, which remains unverified. Physical hardware attribution and direct +Media Foundation capture integration are later work; this change does not +silently claim either one. + +## Scope + +- No production deployment was made. +- No SFU, TURN, Web viewer, access protocol, or default VP8 behavior changed. +- No codec matrix, benchmark, second viewer, or endurance run is required for + this landing slice. +- H.264 distribution still carries the existing patent/license review boundary. + +--- diff --git a/docs/status.md b/docs/status.md index f00210f..687a6c2 100644 --- a/docs/status.md +++ b/docs/status.md @@ -36,10 +36,10 @@ automatic peer/SFU-UDP controller with at most two SFU roots. The old - Host A+B and authenticated P2P Viewer C are sanitized, generation-bound, read-only, and fail closed for stale, ambiguous, or SFU-fed evidence; no raw media metadata is retained. - SVC is `no-go-web-svc-cross-path-hardware-contract`: no direct/peer selection, cross-PC shared encode, or portable hardware proof; pinned screen share is `L1T3`. No browser run was warranted. - The `a11a73d` built-in TURN canary is rejected: local allocation passed, but direct Host/Pion Viewer failed before forced relay. Full rollback restored STUN-only client ICE and zero allocations. -- Native VP8 loopback (Chrome 151) reached one Viewer: 30 sender frames, 85 source RTP packets, 877 inbound packets, 299 decoded/rendered at 1280x720, no fatal. Pion outbound delta missed a 2s refresh; hardware/multi-viewer/FIFO/endurance/public/native proof remains open. +- Native loopbacks (Chrome 151) reached 1280x720 Viewers: VP8 30 sender/85 source RTP/877 inbound/299 decoded; H.264 opt-in 18/156/2,690/299 decoded/298 rendered, with no fatal/encoder errors. H.264 uses `42c01f` Annex-B; `prefer-hardware` is not hardware proof. Pion outbound snapshot timing remains; multi-viewer/FIFO/endurance/public/native proof is open, production stays VP8. - Access protocol/config/HTTP/storage/SQLite/signaling focused tests pass, including commit-first teardown and v1 rollback. - Source adds room-1 Host ingress + peer last-mile; Peer ICE STUN-only; 160 tests pass. -- Prepared artifact `screener-61a87ae47e38abc943cef47b3d7318bb51bf86d6-20260820T143538Z.tar.gz` (883597 bytes, SHA-256 `d5b4d6cc567ba96ce37f8eb9934dd5822c64dc084796047d5f6d6217e2e74acb`) was the sole Host-name cutover candidate. Its prepare-to-release and health gates passed before the stale restart-count assertion caused automatic rollback; `fdd14ba` is healthy with all four service restart counts at 0, and Host display-name remains source-only. Selected-edge TURN is disabled. +- The `61a87ae` Host-name artifact was rolled back after a stale restart-count assertion; `fdd14ba` is healthy and Host display-name remains source-only. Selected-edge TURN is disabled. ## Unverified Boundaries @@ -48,7 +48,7 @@ automatic peer/SFU-UDP controller with at most two SFU roots. The old - Silent partitions can wait 30 to 60 seconds for heartbeat detection before the default 5-second grace; this remains unverified. - Real audio and heterogeneous clients remain unverified. Production reports poor film audio and voice-call self-echo under system capture; there is no app audio ceiling or Web process isolation. Diagnose A/B/C and sync, then gate Windows 11 game-process audio; Windows 10 stays unresolved without system fallback. - The former release reportedly sustained bandwidth-limited blur on a capable LAN; Host refresh recovered while Viewer refresh did not. The new deployment has not yet reproduced or cleared it. -- ADR-0006 has one Viewer decode/render proof; two-edge/FIFO/hardware/endurance/public remain open. +- ADR-0006 has one Viewer proof; two-edge/FIFO, physical hardware, endurance, public, packaged-native, and browser-diversity proof remain open. - Browser fanout is host two/viewer one; any accepted endpoint relay stays capped at two downstream edges. - PR #49/#50 BWE is unverified/default-off: test zero-child leaves, then root impact. No local per-leaf UDP shaper; resume with Linux `tc` or public canary, never CDP. Web P2P/SVC shortcuts remain no-go. - The corrected standby smoke is localhost/headless/video-only; public DNS/TLS reuse, transport, audio, shaping, load, mobile, endurance, and sub-25 ms overlap remain open. @@ -64,7 +64,8 @@ SFU; it never runs three identical retries or abandons progressing P2P early. Next gate exactly-two/Dynacast-off BWE on a zero-child SFU leaf, then root evacuation. Native source accepts increasing capture timestamps shorter than -duration; one VP8 Viewer decode/render run is recorded. Next prove fresh Pion +duration; VP8 and explicit H.264 Viewer decode/render are recorded. Hardware +preference is not physical proof. Next prove fresh Pion outbound diagnostics before viewer 2/FIFO/hardware/endurance/deploy. Audio A/B/C may proceed without displacing P0. Ordinary Peer ICE stays STUN-only; participant-wide TURN is removed. Selected-edge diff --git a/native/sender/internal/app/server.go b/native/sender/internal/app/server.go index f209c97..a9b0dfa 100644 --- a/native/sender/internal/app/server.go +++ b/native/sender/internal/app/server.go @@ -50,6 +50,7 @@ type App struct { session *remote.Session sessionCancel context.CancelFunc room remote.Room + codec media.Codec mediaConn *websocket.Conn mediaClaimed bool mediaClaimID uint64 @@ -63,6 +64,7 @@ type App struct { type startRequest struct { ServerURL string `json:"serverUrl"` Password string `json:"password"` + Codec string `json:"codec,omitempty"` } type encoderConfig struct { @@ -85,6 +87,7 @@ func New() (*App, error) { ctx: ctx, cancel: cancel, token: token, attachTimeout: mediaAttachTimeout, configTimeout: mediaConfigTimeout, + codec: media.CodecVP8, }, nil } @@ -174,6 +177,11 @@ func (app *App) handleStart(response http.ResponseWriter, request *http.Request) writeJSON(response, http.StatusBadRequest, map[string]string{"error": err.Error()}) return } + codec, validCodec := media.ParseCodec(strings.TrimSpace(input.Codec)) + if input.Codec != "" && !validCodec { + writeJSON(response, http.StatusBadRequest, map[string]string{"error": "unsupported native codec"}) + return + } baseURL, err := normalizeRemoteBase(input.ServerURL) if err != nil { writeJSON(response, http.StatusBadRequest, map[string]string{"error": err.Error()}) @@ -196,6 +204,7 @@ func (app *App) handleStart(response http.ResponseWriter, request *http.Request) session, room, err := remote.Start(startContext, remote.StartOptions{ BaseURL: baseURL, HostAdmissionPassword: strings.TrimSpace(input.Password), + Codec: codec, OnEvent: func(event remote.Event) { app.emit(generation, event) }, }) app.mu.Lock() @@ -209,6 +218,7 @@ func (app *App) handleStart(response http.ResponseWriter, request *http.Request) app.session = session app.sessionCancel = startCancel app.room = room + app.codec = codec app.attachTimer = time.AfterFunc(app.attachTimeout, func() { app.mu.Lock() expired := app.generation == generation && app.session == session && app.mediaConn == nil @@ -322,7 +332,7 @@ func (app *App) handleMedia(response http.ResponseWriter, request *http.Request) return } var config encoderConfig - if err = decodeLocalPayload(payload, &config); err != nil || !validEncoderConfig(config) { + if err = decodeLocalPayload(payload, &config); err != nil || !validEncoderConfigForCodec(config, session.Codec()) { session.Fail(errors.New("local encoder configuration is invalid")) _ = connection.Close(websocket.StatusPolicyViolation, "invalid encoder config") return @@ -454,6 +464,7 @@ func (app *App) stopActiveGeneration(expectedGeneration uint64) { app.attachTimer = nil } app.room = remote.Room{} + app.codec = media.CodecVP8 app.activeViewers = 0 app.waitingViewers = 0 app.mu.Unlock() @@ -472,7 +483,11 @@ func (app *App) stopActiveGeneration(expectedGeneration uint64) { } func validEncoderConfig(config encoderConfig) bool { - return config.Kind == "config" && config.Codec == "vp8" && + return validEncoderConfigForCodec(config, media.CodecVP8) +} + +func validEncoderConfigForCodec(config encoderConfig, expected media.Codec) bool { + return config.Kind == "config" && config.Codec == string(expected) && config.Width == 1280 && config.Height == 720 && config.FPS == 30 && config.Bitrate == 3_000_000 && config.EncoderInstances == 1 } diff --git a/native/sender/internal/app/server_test.go b/native/sender/internal/app/server_test.go index 753f9a8..c6926f6 100644 --- a/native/sender/internal/app/server_test.go +++ b/native/sender/internal/app/server_test.go @@ -32,6 +32,19 @@ func TestEncoderConfigUsesTheFixedHighContract(t *testing.T) { } } +func TestEncoderConfigAcceptsExplicitH264OnlyForAnH264Session(t *testing.T) { + config := encoderConfig{ + Kind: "config", Codec: "h264", Width: 1280, Height: 720, + FPS: 30, Bitrate: 3_000_000, EncoderInstances: 1, + } + if !validEncoderConfigForCodec(config, media.CodecH264) { + t.Fatal("explicit H.264 config was rejected") + } + if validEncoderConfigForCodec(config, media.CodecVP8) { + t.Fatal("H.264 config crossed the VP8 session boundary") + } +} + func TestDecodeLocalPayloadIsStrictAndSingleValued(t *testing.T) { valid := []byte(`{"kind":"config","codec":"vp8","width":1280,"height":720,"fps":30,"bitrate":3000000,"encoderInstances":1}`) var config encoderConfig diff --git a/native/sender/internal/app/ui/app.js b/native/sender/internal/app/ui/app.js index 4394b6b..3215f5f 100644 --- a/native/sender/internal/app/ui/app.js +++ b/native/sender/internal/app/ui/app.js @@ -20,6 +20,7 @@ history.replaceState(null, "", location.pathname); const elements = { serverURL: document.querySelector("#server-url"), password: document.querySelector("#password"), + codec: document.querySelector("#codec"), start: document.querySelector("#start"), stop: document.querySelector("#stop"), preview: document.querySelector("#preview"), @@ -81,6 +82,7 @@ async function startSharing() { startInFlight = true; elements.start.disabled = true; const generation = ++startGeneration; + const codec = selectedCodec(); setStatus("选择屏幕"); try { const capturedStream = await navigator.mediaDevices.getDisplayMedia({ @@ -108,6 +110,7 @@ async function startSharing() { const room = await localRequest("/api/start", { serverUrl: elements.serverURL.value, password: elements.password.value, + codec, }); assertCurrentStart(generation); if (pendingStartGeneration === generation) pendingStartGeneration = 0; @@ -117,9 +120,9 @@ async function startSharing() { elements.inviteLink.textContent = room.inviteUrl; elements.invite.hidden = false; - await connectMediaSocket(generation); + await connectMediaSocket(generation, codec); assertCurrentStart(generation); - const activeEncoder = await startEncoder(generation); + const activeEncoder = await startEncoder(generation, codec); assertCurrentStart(generation); sharing = true; elements.stop.hidden = false; @@ -136,7 +139,7 @@ async function startSharing() { } } -async function connectMediaSocket(generation) { +async function connectMediaSocket(generation, codec) { const url = new URL("/media", location.href); url.protocol = "ws:"; const socket = new WebSocket(url, [`screener.token.${processToken}`]); @@ -157,7 +160,7 @@ async function connectMediaSocket(generation) { const accepted = waitForLocalKind(socket, "config-accepted", "本地编码配置确认超时"); socket.send(JSON.stringify({ kind: "config", - codec: "vp8", + codec, width: WIDTH, height: HEIGHT, fps: FPS, @@ -168,9 +171,9 @@ async function connectMediaSocket(generation) { assertCurrentStart(generation, socket); } -async function startEncoder(generation) { +async function startEncoder(generation, codec) { const preferred = { - codec: "vp8", + codec: codec === "h264" ? "avc1.42c01f" : "vp8", width: WIDTH, height: HEIGHT, bitrate: BITRATE, @@ -179,6 +182,7 @@ async function startEncoder(generation) { bitrateMode: "variable", hardwareAcceleration: HARDWARE_PREFERENCE, }; + if (codec === "h264") preferred.avc = { format: "annexb" }; let support = await VideoEncoder.isConfigSupported(preferred); assertCurrentStart(generation, mediaSocket); if (!retainsHardwarePreference(support)) { @@ -504,6 +508,10 @@ function boundedCount(value) { return Number.isSafeInteger(number) && number >= 0 ? number : 0; } +function selectedCodec() { + return elements.codec?.value === "h264" ? "h264" : "vp8"; +} + function emptyMetrics() { return { framesRead: 0, diff --git a/native/sender/internal/app/ui/index.html b/native/sender/internal/app/ui/index.html index 0806c7c..199e897 100644 --- a/native/sender/internal/app/ui/index.html +++ b/native/sender/internal/app/ui/index.html @@ -38,6 +38,13 @@ 访问密码 +

当前 canary 仅发送画面，不含音频。

diff --git a/native/sender/internal/media/fanout.go b/native/sender/internal/media/fanout.go index d6bb809..598cbf7 100644 --- a/native/sender/internal/media/fanout.go +++ b/native/sender/internal/media/fanout.go @@ -21,6 +21,29 @@ const ( videoClockRate = 90_000 ) +// Codec selects the encoded representation carried by the native bridge. +// VP8 remains the default; H.264 is an explicit Windows/WebCodecs opt-in. +type Codec string + +const ( + CodecVP8 Codec = "vp8" + CodecH264 Codec = "h264" + + H264ProfileLevelID = "42c01f" + H264SDPFmtpLine = "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=" + H264ProfileLevelID +) + +func ParseCodec(value string) (Codec, bool) { + switch Codec(value) { + case CodecVP8: + return CodecVP8, true + case CodecH264: + return CodecH264, true + default: + return CodecVP8, false + } +} + type Frame struct { KeyFrame bool TimestampMicros uint64 @@ -44,6 +67,7 @@ type rtpWriter interface { type Fanout struct { track *webrtc.TrackLocalStaticRTP writer rtpWriter + codec Codec queue *frameQueue ctx context.Context cancel context.CancelFunc @@ -57,22 +81,44 @@ type Fanout struct { } func NewFanout(requestKeyFrame func(), onFatal func(error)) (*Fanout, error) { - track, err := webrtc.NewTrackLocalStaticRTP( - webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8, ClockRate: videoClockRate}, - "screen", - "screener", - ) + return NewFanoutWithCodec(CodecVP8, requestKeyFrame, onFatal) +} + +func NewFanoutWithCodec(codec Codec, requestKeyFrame func(), onFatal func(error)) (*Fanout, error) { + capability, err := trackCapability(codec) if err != nil { - return nil, fmt.Errorf("create shared VP8 track: %w", err) + return nil, err + } + track, err := webrtc.NewTrackLocalStaticRTP(capability, "screen", "screener") + if err != nil { + return nil, fmt.Errorf("create shared %s track: %w", codec, err) + } + return newFanoutWithCodec(track, track, codec, requestKeyFrame, onFatal), nil +} + +func trackCapability(codec Codec) (webrtc.RTPCodecCapability, error) { + switch codec { + case CodecVP8: + return webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8, ClockRate: videoClockRate}, nil + case CodecH264: + return webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeH264, ClockRate: videoClockRate, SDPFmtpLine: H264SDPFmtpLine, + }, nil + default: + return webrtc.RTPCodecCapability{}, fmt.Errorf("unsupported native codec %q", codec) } - return newFanout(track, track, requestKeyFrame, onFatal), nil } func newFanout(track *webrtc.TrackLocalStaticRTP, writer rtpWriter, requestKeyFrame func(), onFatal func(error)) *Fanout { + return newFanoutWithCodec(track, writer, CodecVP8, requestKeyFrame, onFatal) +} + +func newFanoutWithCodec(track *webrtc.TrackLocalStaticRTP, writer rtpWriter, codec Codec, requestKeyFrame func(), onFatal func(error)) *Fanout { ctx, cancel := context.WithCancel(context.Background()) fanout := &Fanout{ track: track, writer: writer, + codec: codec, queue: newFrameQueue(queueCapacity), ctx: ctx, cancel: cancel, @@ -90,7 +136,7 @@ func (fanout *Fanout) Track() *webrtc.TrackLocalStaticRTP { func (fanout *Fanout) Push(frame Frame) error { if len(frame.Data) == 0 || len(frame.Data) > MaxFrameBytes || frame.DurationMicros == 0 { - return errors.New("encoded VP8 frame violates the bounded media contract") + return errors.New("encoded frame violates the bounded media contract") } result, err := fanout.queue.Push(frame) if err != nil { @@ -117,7 +163,7 @@ func (fanout *Fanout) Close() { func (fanout *Fanout) writeLoop() { defer close(fanout.done) - timeline := newFrameTimeline() + timeline := newFrameTimelineForCodec(fanout.codec) for { frame, ok, err := fanout.queue.Pop(fanout.ctx) if err != nil { @@ -180,11 +226,21 @@ type frameTimeline struct { } func newFrameTimeline() *frameTimeline { + return newFrameTimelineForCodec(CodecVP8) +} + +func newFrameTimelineForCodec(codec Codec) *frameTimeline { + var payloader rtp.Payloader + if codec == CodecH264 { + payloader = &codecs.H264Payloader{} + } else { + payloader = &codecs.VP8Payloader{} + } return &frameTimeline{packetizer: rtp.NewPacketizer( 1200, 96, 0, - &codecs.VP8Payloader{}, + payloader, rtp.NewRandomSequencer(), videoClockRate, )} diff --git a/native/sender/internal/media/fanout_test.go b/native/sender/internal/media/fanout_test.go index 869cc6e..e80e05d 100644 --- a/native/sender/internal/media/fanout_test.go +++ b/native/sender/internal/media/fanout_test.go @@ -9,6 +9,7 @@ import ( "time" "github.com/pion/rtp" + "github.com/pion/webrtc/v4" ) func TestFrameTimelinePreservesDroppedSourceTime(t *testing.T) { @@ -99,6 +100,38 @@ func TestFrameTimelineAdvancesForPositiveSubsampleDelta(t *testing.T) { } } +func TestH264TimelinePacketizesAnnexBWithInBandParameterSets(t *testing.T) { + timeline := newFrameTimelineForCodec(CodecH264) + accessUnit := []byte{ + 0, 0, 0, 1, 0x09, 0xf0, + 0, 0, 0, 1, 0x67, 0x42, 0xc0, 0x1f, + 0, 0, 0, 1, 0x68, 0xce, 0x06, 0xe2, + 0, 0, 0, 1, 0x65, 0x88, 0x84, 0x21, + } + packets, _, err := timeline.Packetize(Frame{ + KeyFrame: true, TimestampMicros: 1, DurationMicros: 33_333, Data: accessUnit, + }) + if err != nil { + t.Fatal(err) + } + if len(packets) < 2 { + t.Fatalf("H.264 access unit yielded %d RTP packets", len(packets)) + } + if packets[0].Payload[0]&0x1f != 24 { + t.Fatalf("first H.264 payload type = %d, want STAP-A (24)", packets[0].Payload[0]&0x1f) + } +} + +func TestH264TrackCapabilityUsesFixtureProfile(t *testing.T) { + capability, err := trackCapability(CodecH264) + if err != nil { + t.Fatal(err) + } + if capability.MimeType != webrtc.MimeTypeH264 || capability.SDPFmtpLine != H264SDPFmtpLine { + t.Fatalf("H.264 capability = %+v", capability) + } +} + func TestFanoutPropagatesWriterFailure(t *testing.T) { written := make(chan struct{}, 1) fatal := make(chan error, 1) diff --git a/native/sender/internal/remote/codec.go b/native/sender/internal/remote/codec.go new file mode 100644 index 0000000..22ad94d --- /dev/null +++ b/native/sender/internal/remote/codec.go @@ -0,0 +1,56 @@ +package remote + +import ( + "fmt" + + "github.com/TNTcraftHIM/Screener/native/sender/internal/media" + "github.com/pion/webrtc/v4" +) + +const ( + h264PayloadType = 114 + h264RTXPayloadType = 115 +) + +// newPeerAPI keeps the default VP8 API untouched. H.264 gets a private +// MediaEngine so its fixture-derived profile is advertised only on opt-in +// sessions and cannot alter Web or ordinary Native negotiation. +func newPeerAPI(codec media.Codec) (*webrtc.API, []webrtc.RTPCodecParameters, error) { + if codec == media.CodecVP8 { + return nil, nil, nil + } + if codec != media.CodecH264 { + return nil, nil, fmt.Errorf("unsupported native codec %q", codec) + } + + engine := &webrtc.MediaEngine{} + if err := engine.RegisterDefaultCodecs(); err != nil { + return nil, nil, fmt.Errorf("register default codecs: %w", err) + } + feedback := []webrtc.RTCPFeedback{ + {Type: "ccm", Parameter: "fir"}, + {Type: "nack"}, + {Type: "nack", Parameter: "pli"}, + } + primary := webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeH264, ClockRate: 90_000, + SDPFmtpLine: media.H264SDPFmtpLine, RTCPFeedback: feedback, + }, + PayloadType: h264PayloadType, + } + rtx := webrtc.RTPCodecParameters{ + RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeRTX, ClockRate: 90_000, + SDPFmtpLine: fmt.Sprintf("apt=%d", h264PayloadType), + }, + PayloadType: h264RTXPayloadType, + } + if err := engine.RegisterCodec(primary, webrtc.RTPCodecTypeVideo); err != nil { + return nil, nil, fmt.Errorf("register H.264 codec: %w", err) + } + if err := engine.RegisterCodec(rtx, webrtc.RTPCodecTypeVideo); err != nil { + return nil, nil, fmt.Errorf("register H.264 RTX codec: %w", err) + } + return webrtc.NewAPI(webrtc.WithMediaEngine(engine)), []webrtc.RTPCodecParameters{primary, rtx}, nil +} diff --git a/native/sender/internal/remote/codec_test.go b/native/sender/internal/remote/codec_test.go new file mode 100644 index 0000000..1df62d7 --- /dev/null +++ b/native/sender/internal/remote/codec_test.go @@ -0,0 +1,66 @@ +package remote + +import ( + "strings" + "testing" + + "github.com/TNTcraftHIM/Screener/native/sender/internal/media" + "github.com/pion/webrtc/v4" +) + +func TestH264PeerAPIRegistersFixtureProfileAndRTX(t *testing.T) { + api, preferences, err := newPeerAPI(media.CodecH264) + if err != nil { + t.Fatal(err) + } + if api == nil || len(preferences) != 2 { + t.Fatalf("H.264 API/preferences = %v/%d", api != nil, len(preferences)) + } + if preferences[0].MimeType != webrtc.MimeTypeH264 || + preferences[0].SDPFmtpLine != media.H264SDPFmtpLine || + preferences[1].MimeType != webrtc.MimeTypeRTX || preferences[1].SDPFmtpLine != "apt=114" { + t.Fatalf("H.264 preferences = %+v", preferences) + } + if defaultAPI, defaultPreferences, err := newPeerAPI(media.CodecVP8); err != nil || defaultAPI != nil || defaultPreferences != nil { + t.Fatalf("VP8 unexpectedly changed API boundary: %v/%v/%v", defaultAPI, defaultPreferences, err) + } +} + +func TestH264PeerAPIOffersFixtureProfileWhenTrackIsBound(t *testing.T) { + api, preferences, err := newPeerAPI(media.CodecH264) + if err != nil { + t.Fatal(err) + } + fanout, err := media.NewFanoutWithCodec(media.CodecH264, nil, nil) + if err != nil { + t.Fatal(err) + } + defer fanout.Close() + connection, err := api.NewPeerConnection(webrtc.Configuration{}) + if err != nil { + t.Fatal(err) + } + defer connection.Close() + if _, err = connection.AddTrack(fanout.Track()); err != nil { + t.Fatal(err) + } + if err = connection.GetTransceivers()[0].SetCodecPreferences(preferences); err != nil { + t.Fatal(err) + } + offer, err := connection.CreateOffer(nil) + if err != nil { + t.Fatal(err) + } + if !containsAll(offer.SDP, []string{"H264/90000", "profile-level-id=42c01f", "packetization-mode=1"}) { + t.Fatalf("H.264 offer omitted fixture profile: %s", offer.SDP) + } +} + +func containsAll(value string, needles []string) bool { + for _, needle := range needles { + if !strings.Contains(value, needle) { + return false + } + } + return true +} diff --git a/native/sender/internal/remote/peer.go b/native/sender/internal/remote/peer.go index 3d6a192..e0f46cf 100644 --- a/native/sender/internal/remote/peer.go +++ b/native/sender/internal/remote/peer.go @@ -10,6 +10,7 @@ import ( "sync/atomic" "time" + "github.com/TNTcraftHIM/Screener/native/sender/internal/media" "github.com/pion/rtcp" "github.com/pion/webrtc/v4" ) @@ -54,7 +55,12 @@ func newPeer(session *Session, peerID string, slot int, configuration webrtc.Con if err != nil { return nil, fmt.Errorf("create connection identity: %w", err) } - connection, err := webrtc.NewPeerConnection(configuration) + var connection *webrtc.PeerConnection + if session.peerAPI != nil { + connection, err = session.peerAPI.NewPeerConnection(configuration) + } else { + connection, err = webrtc.NewPeerConnection(configuration) + } if err != nil { return nil, errors.New("create viewer peer connection failed") } @@ -94,7 +100,15 @@ func newPeer(session *Session, peerID string, slot int, configuration webrtc.Con sender, err := connection.AddTrack(session.fanout.Track()) if err != nil { _ = connection.Close() - return nil, errors.New("bind shared VP8 track failed") + return nil, errors.New("bind shared encoded track failed") + } + if session.codec == media.CodecH264 { + transceivers := connection.GetTransceivers() + if len(transceivers) != 1 || len(session.codecPreferences) == 0 || + transceivers[0].SetCodecPreferences(session.codecPreferences) != nil { + _ = connection.Close() + return nil, errors.New("configure H.264 codec preferences failed") + } } go peer.readRTCP(sender) return peer, nil diff --git a/native/sender/internal/remote/session.go b/native/sender/internal/remote/session.go index 543fde7..af2d940 100644 --- a/native/sender/internal/remote/session.go +++ b/native/sender/internal/remote/session.go @@ -53,6 +53,7 @@ type Event struct { type StartOptions struct { BaseURL *url.URL HostAdmissionPassword string + Codec media.Codec OnEvent func(Event) } @@ -74,20 +75,23 @@ type Session struct { shareGeneration string onEvent func(Event) - mu sync.Mutex - writeMu sync.Mutex - conn *websocket.Conn - ice iceConfig - peers map[string]*peer - admission *viewerAdmission - fanout *media.Fanout - closeOnce sync.Once - fatalOnce sync.Once - terminalOnce sync.Once - readDone chan struct{} - terminalResult chan error - readerStarted atomic.Bool - closing atomic.Bool + mu sync.Mutex + writeMu sync.Mutex + conn *websocket.Conn + ice iceConfig + peers map[string]*peer + admission *viewerAdmission + fanout *media.Fanout + codec media.Codec + peerAPI *webrtc.API + codecPreferences []webrtc.RTPCodecParameters + closeOnce sync.Once + fatalOnce sync.Once + terminalOnce sync.Once + readDone chan struct{} + terminalResult chan error + readerStarted atomic.Bool + closing atomic.Bool } type createRoomResponse struct { @@ -121,6 +125,13 @@ func Start(parent context.Context, options StartOptions) (*Session, Room, error) if options.BaseURL == nil { return nil, Room{}, errors.New("remote server URL is required") } + codec := options.Codec + if codec == "" { + codec = media.CodecVP8 + } + if _, ok := media.ParseCodec(string(codec)); !ok { + return nil, Room{}, fmt.Errorf("unsupported native codec %q", codec) + } jar, err := cookiejar.New(nil) if err != nil { return nil, Room{}, fmt.Errorf("create in-memory cookie jar: %w", err) @@ -160,12 +171,13 @@ func Start(parent context.Context, options StartOptions) (*Session, Room, error) hostToken: created.HostToken, clientID: clientID, shareGeneration: shareGeneration, + codec: codec, onEvent: options.OnEvent, peers: make(map[string]*peer), readDone: make(chan struct{}), terminalResult: make(chan error, 1), } - session.fanout, err = media.NewFanout( + session.fanout, err = media.NewFanoutWithCodec(codec, func() { session.emit(Event{Kind: "request-keyframe"}) }, func(fatalErr error) { session.fail(fatalErr) }, ) @@ -173,6 +185,11 @@ func Start(parent context.Context, options StartOptions) (*Session, Room, error) cancel() return nil, Room{}, err } + session.peerAPI, session.codecPreferences, err = newPeerAPI(codec) + if err != nil { + session.close() + return nil, Room{}, err + } if err = session.connectWithRetry(parent); err != nil { session.close() return nil, Room{}, err @@ -219,6 +236,10 @@ func (session *Session) WriteFrame(frame media.Frame) error { return nil } +func (session *Session) Codec() media.Codec { + return session.codec +} + func (session *Session) Stop() { session.close() } diff --git a/scripts/native-one-viewer-gate.ts b/scripts/native-one-viewer-gate.ts index b24fb2b..5a3b4d2 100644 --- a/scripts/native-one-viewer-gate.ts +++ b/scripts/native-one-viewer-gate.ts @@ -226,13 +226,13 @@ function observeNodeStages(server: ScreenerServer, ledger: SenderStartLedger): v }); } -function senderStartObservation(snapshot: SenderSnapshot): Partial { +function senderStartObservation(snapshot: SenderSnapshot, expectedCodec = "vp8"): Partial { return { getDisplayMediaRequested: snapshot.getDisplayMediaRequested, getDisplayMediaResolved: snapshot.getDisplayMediaResolved, bridgeConnected: snapshot.ready > 0, fixedHighConfigured: snapshot.configAccepted > 0 && - snapshot.config?.codec === "vp8" && snapshot.config.width === 1280 && + snapshot.config?.codec === expectedCodec && snapshot.config.width === 1280 && snapshot.config.height === 720 && snapshot.config.fps === 30 && snapshot.config.bitrate === 3_000_000 && snapshot.config.encoderInstances === 1, firstEncodedChunk: snapshot.encoderOutputs > 0, @@ -273,6 +273,7 @@ function appendLedgerRecord(path: string, record: object): void { async function main(): Promise { const chromePath = requiredEnvironmentPath("SCREENER_NATIVE_GATE_CHROME"); const goPath = process.env.SCREENER_NATIVE_GATE_GO?.trim() || "go"; + const gateCodec = process.env.SCREENER_NATIVE_GATE_CODEC?.trim() === "h264" ? "h264" : "vp8"; const accessKey = randomBytes(9).toString("base64url"); const report: GateReport = { schemaVersion: 1, @@ -369,10 +370,11 @@ async function main(): Promise { await evaluate(cdp, activeSenderPage, `(() => { document.querySelector('#server-url').value = ${JSON.stringify(baseUrl)}; document.querySelector('#password').value = ${JSON.stringify(accessKey)}; + document.querySelector('#codec').value = ${JSON.stringify(gateCodec)}; document.querySelector('#start').click(); })()`, Date.now() + 5_000); await waitForSenderStart( - async (deadline) => senderStartObservation(await senderSnapshot(cdp!, activeSenderPage, deadline)), + async (deadline) => senderStartObservation(await senderSnapshot(cdp!, activeSenderPage, deadline), gateCodec), senderLedger, { timeoutMs: 20_000 }, ); @@ -498,7 +500,7 @@ async function main(): Promise { finalSenderEvidence = await verifyFinalSenderEvidence( () => senderSnapshot(finalCdp, finalSenderPage, Date.now() + 1_000), (snapshot) => { - senderLedger.record(senderStartObservation(snapshot)); + senderLedger.record(senderStartObservation(snapshot, gateCodec)); return senderLedger.snapshot(); }, ); diff --git a/tests/native-hardware-status.test.ts b/tests/native-hardware-status.test.ts index 09ff2a9..95e5ceb 100644 --- a/tests/native-hardware-status.test.ts +++ b/tests/native-hardware-status.test.ts @@ -12,6 +12,10 @@ const senderUI = readFileSync( join(import.meta.dirname, "../native/sender/internal/app/ui/app.js"), "utf8", ); +const senderHTML = readFileSync( + join(import.meta.dirname, "../native/sender/internal/app/ui/index.html"), + "utf8", +); const senderServer = readFileSync( join(import.meta.dirname, "../native/sender/internal/app/server.go"), "utf8", @@ -30,8 +34,10 @@ describe("native sender hardware preference contract", () => { expect(retainsHardwarePreference(null)).toBe(false); }); - it("keeps VP8, one encoder, and an explicit unverified hardware boundary", () => { - expect(senderUI).toContain('codec: "vp8"'); + it("keeps VP8 default, one encoder, and an explicit unverified hardware boundary", () => { + expect(senderHTML).toContain(''); + expect(senderUI).toContain('elements.codec?.value === "h264" ? "h264" : "vp8"'); + expect(senderUI).toContain('codec: codec === "h264" ? "avc1.42c01f" : "vp8"'); expect(senderUI).toContain("encoderInstances: 1"); expect(senderUI).toContain("HARDWARE_STATUS.PREFERENCE_ACCEPTED"); expect(senderUI).toContain("HARDWARE_STATUS.FALLBACK");