diff --git a/.github/workflows/native-sender-package.yml b/.github/workflows/native-sender-package.yml index 69af246..2dadd58 100644 --- a/.github/workflows/native-sender-package.yml +++ b/.github/workflows/native-sender-package.yml @@ -5,7 +5,7 @@ on: pull_request: paths: - "native/sender/**" - - "native/process-audio-helper/**" + - "native/window-capture-helper/**" - "scripts/package-native-sender.ps1" - ".github/workflows/native-sender-package.yml" push: @@ -13,7 +13,7 @@ on: - main paths: - "native/sender/**" - - "native/process-audio-helper/**" + - "native/window-capture-helper/**" - "scripts/package-native-sender.ps1" - ".github/workflows/native-sender-package.yml" diff --git a/docs/adr/0008-window-scoped-audio-capture.md b/docs/adr/0008-window-scoped-audio-capture.md index e089d44..a344e67 100644 --- a/docs/adr/0008-window-scoped-audio-capture.md +++ b/docs/adr/0008-window-scoped-audio-capture.md @@ -2,9 +2,10 @@ Date: 2026-08-21 -Status: Accepted. The browser hint, opt-in Native Windows P1 slice, and bounded -Windows x64 evaluation-package path are source-complete; Native remains -default-off, formally unreleased, and undeployed. +Status: Accepted. The browser hint and opt-in Native Windows audio slice are +source-complete, including the bounded Windows x64 evaluation-package path. +The paired WGC/MF hardware-video source passed its one-Viewer acceptance smoke; +all Native sources remain default-off, formally unreleased, and undeployed. ## Context @@ -33,12 +34,12 @@ Windows 11 desktop (build 22000 or newer) and is not a Windows 10 fallback. 2. The default Web route remains unchanged in topology, codecs, and relay behavior. No Web Audio mixer, SDP rewrite, or app-owned audio bitrate knob is introduced by this ADR. -3. The native sender's next audio slice is an explicit opt-in - `window-process-audio` mode on Windows 11. The local capture session owns - one selected target PID/process tree, one WGC/DXGI video source, and one - WASAPI loopback audio source. Process IDs, creation tokens, titles, paths, - device identity, and PCM stay local; they do not enter signaling, URLs, - logs, or storage. +3. The native sender has an explicit Windows 11 local window target. Browser + VP8/H.264 may use that target only for `window-process-audio`; the separate + `native-window-h264` source uses the same bound HWND/PID/creation time for + WGC/DXGI video and WASAPI process-tree audio. Process IDs, HWNDs, creation + tokens, titles, paths, device identity, PCM, and local hardware status stay + local; they do not enter signaling, URLs, logs, or storage. 4. If the target process exits, produces no render stream, or the loopback activation is denied, the sender reports `audio unavailable` or `audio silent` and stops/asks the user. It never widens to whole-system audio. @@ -50,7 +51,7 @@ Windows 11 desktop (build 22000 or newer) and is not a Windows 10 fallback. This is an interface boundary, not a new framework or a current wire change: ```text -NativeCaptureTarget { pid: uint32, creationTime: uint64, includeProcessTree: true } +NativeCaptureTarget { hwnd: uint64, pid: uint32, creationTime: uint64, includeProcessTree: true } NativeCaptureSession.start(target) -> VideoFrames(WGC/DXGI) + AudioPcm(WASAPI process loopback) -> existing native sender timeline/encoder/fanout @@ -80,9 +81,9 @@ timestamp, duration, bounded payload); it does not add a public signaling field. The Go session owns one shared Opus `TrackLocalStaticRTP` and adds it before the offer on every native PeerConnection. Audio packetization uses the 48 kHz RTP clock and the source timestamp delta, so one encoded packet stream can feed the -existing two-edge cap. The WGC/DXGI video adapter remains the next small slice; -its `SystemRelativeTime` is retained in the fixture so it can share the same -QPC origin when native video replaces the Web source. +existing two-edge cap. In this historical audio-only phase, the WGC/DXGI video +adapter was the next small slice; its retained `SystemRelativeTime` let the +completed native-video path share the same QPC origin. Expected implementation size is about **500--800 new LOC**, excluding the browser/Windows SDK and existing video/MF fixture code: Windows loopback @@ -112,11 +113,29 @@ visual marker, while an independent voice process and notification emit different markers. Require an Opus inbound track and rendered video/audio, monotonic QPC timestamps, and a bounded A/V offset; verify the unrelated markers are absent and that target exit/silence stops or asks without widening -capture. SFU/TURN, second Viewer, endurance, and the native WGC video swap are -follow-up gates. On 2026-08-21, 100 20-ms chunks isolated a 440 Hz target from +capture. SFU/TURN, second Viewer, and endurance remain follow-up gates; paired +WGC video completion is recorded below. On 2026-08-21, 100 20-ms chunks isolated a 440 Hz target from an independent 880 Hz process by 4017.8x. Chrome 151 then received one audio track and 495 inbound Opus packets while video decoded/rendered 296 frames. +## P1 WGC/MF Completion + +The paired native-video source reuses the proven hardware fixture rather than +adding a second encoder framework. WGC creates a capture item directly from the +bound HWND on the selected D3D11 device. The frame pool is free-threaded and is +recreated on content-size changes. D3D11 VideoProcessor performs aspect-fit +BGRA-to-NV12 conversion at fixed 1280x720/30, then the existing adapter-bound, +hardware-only asynchronous MF H.264 path emits Annex-B `42c01f` access units. +There is no software MFT, alternate codec, monitor, or browser-video fallback. + +The helper multiplexes process-tree PCM, H.264 access units, and bounded local +status. Go writes H.264 directly to the existing shared fanout; the browser UI +receives only a one-way local copy for `VideoDecoder` preview. PCM retains the +existing WebCodecs Opus bridge. PLI/FIR recovery is generation-bound and +coalesced into a helper key-frame command. Normal browser VP8/H.264 sources +remain explicit alternatives, with VP8 as the default; neither is an implicit +runtime downgrade. + ## Consequences - Current Web users get a useful Chromium hint and an honest status without a @@ -131,8 +150,8 @@ track and 495 inbound Opus packets while video decoded/rendered 296 frames. ## Follow-Up TODO +- Keep a second Viewer, SFU/TURN, endurance, and real-game A/V sync as separate + follow-ups; the one-Viewer H.264 and attributed `VideoEncode` gate passed. - Download and run the short-lived Windows x64 evaluation artifact; keep explicit target selection default-off. A formal release remains blocked on the project license, signing, and its own acceptance boundary. -- Keep native WGC video, a second Viewer, SFU/TURN, endurance, and real-game - A/V sync as separate follow-ups. diff --git a/docs/project-memory.md b/docs/project-memory.md index 388f645..3565c42 100644 --- a/docs/project-memory.md +++ b/docs/project-memory.md @@ -5,7 +5,7 @@ Last updated: 2026-08-21 ## Confirmed Intent - Build low-latency game sharing for one broadcaster and trusted friends; public or large broadcasts belong on OBS/Twitch-class services. -- Viewers join from normal desktop/mobile browsers. A later native sender may share encoding and improve capture/audio without changing the Web viewer requirement. +- Viewers join from normal desktop/mobile browsers. Native senders may improve capture/audio without changing the Web viewer requirement. - Use a small central service for access, rooms, signaling, deterministic topology, STUN, observability, and bounded SFU-root capacity. - Minimize server bandwidth. Ordinary peer ICE is STUN-only. Failure tries restart, rebuild, alternate peer and bounded SFU/UDP roots; only then may the controller authorize short-lived TURN for one selected exceptional edge before clear failure. TURN is transport, not topology. - Non-server nodes have at most two downstream edges; browser relays stay at one until resource gates pass. SFU normally feeds one or two roots that retain peer descendants; separately capped server edges may serve exceptional viewers that cannot attach behind a healthy root. @@ -23,7 +23,7 @@ Last updated: 2026-08-21 - Use direct host P2P for one or two viewers, then the bounded controller for every room when `PEER_ASSISTED_MEDIA=true`. Room `1` is historical smoke, not a runtime gate; resource, quality, recovery, SFU/UDP, bounded-failure, and browser/mobile evidence remain separate. - Browser relays resend remote `MediaStreamTrack` values and re-encode at each hop; WebRTC does not guarantee a shared encoder across peer connections, so measure the cost. - Keep experiments bounded: the standard representation sequence is below; native RTP relay, encoded-object striping, and FEC remain separate. -- ADR-0006 remains no-go. VP8 is default; H.264 loopback rendered 298/299. Windows x64 evaluation packaging is source-complete; downloaded use, Pion timing, hardware, multi-viewer/endurance/public proof remain open. +- ADR-0006's historical browser-bridge canary remains no-go; VP8 stays default. Browser H.264 rendered 298/299; Native WGC/MF rendered 203 with PID/LUID-correlated `VideoEncode`. Downloaded use, multi-viewer/endurance/public proof remain open. - Mobile Web Host unsupported. Android 14+ sender is direct-child video-only; device, audio, rotation, and SFU remain open. iOS deferred; TV output is local-only P2. - ADR-0005 accepts direct/peer UDP, bounded SFU roots, then optional selected-edge TURN. Production now enables the one-shot selected-edge tuple after SFU; participant-wide TURN remains rejected and ordinary peer ICE stays STUN-only. No real TURN/SFU media canary has yet been run. - The controller is process-enabled, not room-allowlisted: `PEER_ASSISTED_MEDIA=true` gives every normal room the same direct/peer -> SFU -> selected-edge path with per-room state. Room `1` is historical smoke. Preserve sticky P2P, mobile leaves and break-before-make; recovery gets one attempt per layer (ICE restart, same-parent rebuild, alternate peer, then SFU), never three identical retries. @@ -34,7 +34,6 @@ Last updated: 2026-08-21 - Viewer-local names and the opt-in Web participant roster remain control-plane-only. Web Host names are deployed in the current release and stay socket/localStorage-only; Native remains outside the capability boundary. - Do not add scene detection, dynamic-FPS control, or forced AV1 without negotiation, encode, game, CPU/GPU, and sender evidence. - ADR-0002 grant/public access, the v3 room-password migration, and Web Host display names are deployed. Names/presence remain session-only without account or roster tables. -- Deferred architecture audit: `docs/maintenance.md`. ## Current Execution Principle @@ -53,7 +52,7 @@ Last updated: 2026-08-21 - Chrome 151 synthetic topology/quality runs kept fanout 2/1 and decoding; one relay close recovered in 5.32 seconds. Control evidence only. - Room `1` publishes `HIGH+LOW` with Dynacast off and subscriber `HIGH` ceilings, and now has bounded local quality reparenting. Both remain unverified on real media; test zero-child leaves and calibrated C+B loss before broad rollout. - C+B uses three hard-bad pairs, one-use samples and guarded intent/cooldown. Deployed admission rescue moves the oldest childless zero-capacity Host leaf below an unassigned one-slot relay in one revision; no score/timer/global rebalance. Deployment passed, but no real room triggered it. -- Web Host name/presence and window-audio hint are deployed. Native source accepts authenticated peer-assisted direct-child assignments, ignores presence as an edge authority, and reports unsupported SFU/selected ingress as bounded route failure; it does not implement LiveKit or TURN media. Native H.264 source and Windows x64 evaluation packaging are present, but no Native release/deployment exists; VP8 remains the Web/production default. See `docs/research/native-h264-opt-in-path.md`. +- Web Host name/presence and window-audio hint are deployed. Native accepts bounded peer-assisted direct children and fails unsupported SFU/selected ingress; it has no LiveKit/TURN media. Its WGC/MF H.264 direct-child smoke rendered 203 frames plus 500 Opus packets and matched helper PID/LUID `VideoEncode`; packaging is source-only, unreleased and undeployed. VP8 remains the Web/production default. See `docs/research/native-h264-opt-in-path.md`. ## Provisional Quality Targets @@ -69,7 +68,7 @@ These are measurement gates, not performance claims. - Sustainable count by hardware, quality, network, and route: finish instrumented `1/3/5/8`, then pass a 20-viewer matrix before changing the accepted target default to 20. - Whether ADR-0004 passes fanout, re-encoding, depth-four latency, reparenting, silent-partition, and mobile-leaf gates. - Whether source-complete selected-edge passes forced relay, expiry/failure, mobile, 1-GiB resource and relay-bandwidth gates. Coturn's bearer is non-revocable until expiry; application guards and TTL only bound reuse. Performance comparison and the retained matrix follow functional landing; Media TCP is out of scope. -- Exact mobile lifecycle behavior and whether Windows per-application audio is required for the first release. +- Exact mobile lifecycle behavior. - Project license and distribution model, which determines whether GPL/AGPL sources can move beyond study-only use. - Initial deployment regions and expected mainland China, Hong Kong, and overseas network mix. - Whether voice chat ever enters scope or Screener stays complementary to an existing voice application. diff --git a/docs/research/native-h264-hardware-decision.md b/docs/research/native-h264-hardware-decision.md index 6b81e4a..85a3ece 100644 --- a/docs/research/native-h264-hardware-decision.md +++ b/docs/research/native-h264-hardware-decision.md @@ -2,16 +2,21 @@ Date: 2026-08-20 -Status: `no-go-native-h264-hardware-pinned-fmtp` +Status: Historical gate. Its `no-go-native-h264-hardware-pinned-fmtp` +stop resolved when the explicit opt-in Pion path registered exact `42c01f`; +the product WGC/MF path later passed its attributed one-Viewer smoke. The later source-only functional slice in [`native-h264-opt-in-path.md`](./native-h264-opt-in-path.md) registers the -fixture profile explicitly and proves one H.264 Host/Viewer loopback. It does -not change this spike's physical-hardware decision: WebCodecs hardware -attribution remains unverified and production remains VP8/default. +fixture profile explicitly and proves one browser H.264 Host/Viewer loopback. +The native-window source now compiles this fixture's MF encoder source into the +product helper, retaining hardware-only enumeration and exact readback. That +path rendered 203 frames for one Chrome Viewer while the exact helper PID and +adapter LUID matched nonzero Windows `VideoEncode`; offline fixture telemetry +remains separate evidence. -This spike does not amend ADR-0006 or authorize another product, room, or -production run. It answers one narrow question: can a bounded Windows H.264 +This spike did not amend ADR-0006 or authorize another product, room, or +production run. It answered one narrow question: can a bounded Windows H.264 path satisfy the project's physical hardware-encode and existing default Pion fmtp contracts before product integration? @@ -20,7 +25,8 @@ emitted SPS differed from the requested codec string. A subsequent hardware-only Media Foundation run did prove the selected NVIDIA encoder and GPU engine, but emitted exact `profile-level-id=42c01f`, which is absent from the default Pion mode-1 fmtp set. It therefore stopped before Pion, a Viewer, -rooms, or product wiring. Neither result authorizes an H.264 product switch. +rooms, or product wiring. The later opt-in source resolved that exact +registration mismatch without changing the browser VP8 default. ## Decision Summary @@ -31,8 +37,8 @@ rooms, or product wiring. Neither result authorizes an H.264 product switch. | Did the requested codec string describe the emitted SPS exactly? | No. The requested `42e01f` produced SPS bytes `42 04 1f`. | | Did forced key frames carry in-band recovery data? | Yes in this fixture: all six key chunks contained AUD, SPS, PPS, and IDR NAL units. | | Can Pion v1.10.5 packetize the resulting Annex-B access units? | Yes statically: its `H264Payloader` splits 3/4-byte start codes, suppresses AUD/filler, emits SPS/PPS as STAP-A, and fragments slices as FU-A. | -| Is unmodified browser Viewer decode proven? | No. No PeerConnection, room, Viewer, or production service participated. | -| Should the small WebCodecs/Pion codec diff be implemented? | No. It would preserve the two unresolved product gates: physical encoder identity and exact emitted-fmtp registration plus Viewer interoperability. | +| Did this isolated spike prove unmodified browser Viewer decode? | No. No PeerConnection, room, Viewer, or production service participated. A later opt-in loopback records that evidence separately. | +| Did this spike authorize the small WebCodecs/Pion codec diff? | No. The later exact-`42c01f` opt-in reopened and proved that boundary separately. | | Did the Media Foundation fixture prove physical H.264 hardware encode? | Yes for one local RTX 4070 SUPER run: adapter-LUID-bound hardware enumeration, D3D11 awareness, the NVIDIA H.264 Encoder MFT, and process-plus-LUID `VideoEncode` activity agreed. | | Did that hardware MFT emit an exact default Pion mode-1 fmtp string? | No. It emitted `42c01f`; the default set is `42001f`, `42e01f`, `4d001f`, and `64001f`. The fixture failed closed without changing the request or adding a fallback. This is not a standards-level profile incompatibility. | @@ -178,10 +184,10 @@ preference, not physical hardware evidence. ## Media Foundation Hardware Fixture The follow-up fixture is retained at -`native/fixtures/mf-h264-hardware/`. It is a Windows-only offline C++ program, -not a sender component. Its generated object and executable stay outside the -repository. It has one H.264 path and no software MFT, codec, adapter, room, -capture, audio, signaling, Pion, or Viewer fallback. +`native/fixtures/mf-h264-hardware/`. Its build compiles the shared encoder +source from `native/window-capture-helper/` with fixture-only synthetic input +and telemetry. The generated fixture executable stays outside the repository +and has no room, capture, audio, signaling, Pion, or Viewer fallback. ### Fixed Contract diff --git a/docs/research/native-h264-opt-in-path.md b/docs/research/native-h264-opt-in-path.md index 7d33135..c46beb5 100644 --- a/docs/research/native-h264-opt-in-path.md +++ b/docs/research/native-h264-opt-in-path.md @@ -1,27 +1,79 @@ # Native H.264 Opt-In Path -## 2026-08-21 00:15 +## 2026-08-21 Native Window Source -The native sender now has a small, explicit H.264 mode. VP8 remains the default -and the Web sender is unchanged. A sender chooses H.264 in its local Codec -control; the choice is carried only through the loopback start/media handshake. +The Native sender now has two explicit local video sources. Browser VP8 remains +the default; browser H.264 remains available as the earlier opt-in. The new +`native-window-h264` source is Windows 11-only and requires the user to select +one local opaque window target. It never silently replaces a browser source or +falls back to software encode, another codec, monitor capture, or system audio. -The H.264 path is: +The native-window path is: ```text -Chrome WebCodecs H.264 Annex-B - -> existing 17-byte local frame envelope +opaque local target -> HWND + PID + process creation time + -> WGC BGRA surface on one selected D3D11 adapter + -> D3D11 VideoProcessor 1280x720 NV12 + -> adapter-bound hardware-only MF H.264 42c01f Annex-B + -> generation-bound Go Session.WriteMedia -> one Go queue and Pion H264Payloader -> one TrackLocalStaticRTP -> at most two independent PeerConnections ``` -The Go session creates a private Pion `MediaEngine` only for H.264 sessions. It -registers the fixture-derived Constrained Baseline `profile-level-id=42c01f`, -`packetization-mode=1`, and a dedicated dynamic payload type, then restricts -the sender transceiver to that H.264 capability. The shared fanout and keyframe -feedback remain unchanged. The default VP8 session still uses Pion's default -API and payloader. +The helper revalidates HWND-to-PID and PID creation time immediately before WGC +activation. The same process identity feeds WASAPI application loopback with +`INCLUDE_TARGET_PROCESS_TREE`. One private stdout protocol multiplexes bounded +PCM, H.264 access units, and local hardware status. PID, HWND, creation time, +adapter/MFT identity, PCM, and status never enter signaling. Go sends H.264 +directly to the existing fanout; a second local copy travels only to the sender +UI's `VideoDecoder` preview and never returns to Go. PCM retains the existing +local WebCodecs Opus bridge. + +PLI/FIR and fanout recovery requests enter the generation-bound helper stdin as +one coalesced `K` command. The helper forces an in-band SPS/PPS/IDR recovery +unit, while the existing Pion `H264Payloader`, exact `42c01f` MediaEngine, one +shared track, and two-edge cap remain unchanged. Browser VP8/H.264 use their +existing `getDisplayMedia` and `VideoEncoder` path. + +Both the product helper and offline fixture compile the same MF encoder source. +The product build and the fixture build pass MSVC `/W4 /WX`; focused Go tests +cover bound target arguments, multiplexed protocol bounds, key-frame command +forwarding, generation/source validation, and the peer-assisted Host contract. + +## Native Window Acceptance Evidence + +One bounded Chrome 151 run used a real animated top-level window, process-tree +audio, the explicit Native source, and the peer-assisted direct-child wire. It +created no browser capture or `VideoEncoder`; the ordinary Viewer received +1,313 packets/1,431,696 bytes during the sample, decoded and rendered 203 +1280x720 frames, and received 500 Opus packets without a fatal/encoder error. +The helper wrote 30 frames and 174 source RTP packets before the retained +sender-start snapshot. + +The exact helper process was PID 15060. Its Windows GPU Engine instance +`pid_15060_luid_0x00000000_0x0001a496_phys_0_eng_6_engtype_videoencode` +reached 2.98028% utilization; the helper's independently validated adapter +status was `0x00000000:0x0001a496`. This is process-and-adapter-correlated +hardware evidence, not a throughput or performance claim. The generic gate +reported failure only because its asynchronous Pion diagnostics delta stayed +zero after the Viewer counters advanced; the run was not repeated for that +known sampling residue. + +Primary sources checked 2026-08-21: + +- [Windows Graphics Capture](https://learn.microsoft.com/en-us/windows/uwp/audio-video-camera/screen-capture) +- [`IGraphicsCaptureItemInterop::CreateForWindow`](https://learn.microsoft.com/en-us/windows/win32/api/windows.graphics.capture.interop/nf-windows-graphics-capture-interop-igraphicscaptureiteminterop-createforwindow) +- [`Direct3D11CaptureFramePool::CreateFreeThreaded`](https://learn.microsoft.com/en-us/uwp/api/windows.graphics.capture.direct3d11captureframepool.createfreethreaded) +- [`CreateDirect3D11DeviceFromDXGIDevice`](https://learn.microsoft.com/en-us/windows/win32/api/windows.graphics.directx.direct3d11.interop/nf-windows-graphics-directx-direct3d11-interop-createdirect3d11devicefromdxgidevice) +- [Microsoft MIT HWND capture sample](https://github.com/microsoft/Windows.UI.Composition-Win32-Samples/tree/master/cpp/ScreenCaptureforHWND) +- [`ID3D11VideoDevice::CreateVideoProcessorInputView`](https://learn.microsoft.com/en-us/windows/win32/api/d3d11/nf-d3d11-id3d11videodevice-createvideoprocessorinputview) +- [`ID3D11VideoContext::VideoProcessorSetOutputBackgroundColor`](https://learn.microsoft.com/en-us/windows/win32/api/d3d11/nf-d3d11-id3d11videocontext-videoprocessorsetoutputbackgroundcolor) + +## 2026-08-21 Browser H.264 Source + +The earlier browser H.264 source remains an explicit option. Its loopback path +is Chrome WebCodecs Annex-B -> the local envelope -> the same Go/Pion fanout. ## Bounded Loopback Evidence @@ -47,21 +99,22 @@ loopback, not a public-network, multi-viewer, endurance, or codec benchmark. ## Hardware Boundary -The Media Foundation/NVIDIA fixture remains the physical hardware evidence and -informed the exact `42c01f` profile and in-band SPS/PPS/keyframe contract. The -product sender requests WebCodecs H.264 hardware acceleration, but browser -`isConfigSupported()` and the preference string do not prove which encoder was -selected. The UI therefore reports hardware preference separately from hardware -evidence, which remains unverified. Physical hardware attribution and direct -Media Foundation capture integration are later work; this change does not -silently claim either one. +The Media Foundation/NVIDIA fixture established the physical hardware gate and +the exact `42c01f` in-band recovery contract. Browser H.264 still has only a +hardware preference. The native-window source instead enumerates only an +adapter-LUID-bound hardware MFT, requires D3D11 awareness and exact codec +readback, and has no software retry. The Native acceptance run above adds the +required product-process `VideoEncode` attribution; fixture evidence remains a +separate encoder-contract measurement. ## Scope -- No production deployment was made. +- Browser VP8 remains the default and no production deployment switch is made. - No SFU, TURN, Web viewer, access protocol, or default VP8 behavior changed. - No codec matrix, benchmark, second viewer, or endurance run is required for this landing slice. +- The Native source reuses existing controller compatibility and evaluation + packaging changes; it adds no topology. - H.264 distribution still carries the existing patent/license review boundary. --- diff --git a/docs/status.md b/docs/status.md index dcc93d6..b3db49b 100644 --- a/docs/status.md +++ b/docs/status.md @@ -7,7 +7,7 @@ Last updated: 2026-08-21 `https://share.bonfire.icu` serves exact `fd76277b05d491af8840b28f3132b7ff445d3cbe`; the 915,377-byte artifact SHA-256 is `8da4b8b2b1ae4b82add615f4367ee447b5ade86c4e58a938e51085940b9867d3`. The cutover held the lock 8,389 ms and returned local health 520 ms after stop. SQLite v3 has five rooms; Screener, LiveKit, coturn and nginx are healthy with zero restarts. Ordinary ICE is STUN-only; all rooms use the bounded peer/SFU-UDP controller and selected-edge UDP TTL 120. Room `1` is historical smoke and no real TURN/SFU media canary has run. -Web Host names/audio hint are deployed. Native H.264 and Windows x64 packaging are source-only; Web defaults VP8. +Web names/audio hint are deployed. Native WGC/MF/package is source-only; Web defaults VP8. ## Execution Principle @@ -18,7 +18,7 @@ Flagship; parallel; min run/smoke/rollback; benchmark later. Active UI/config/lo - Capture precedes room creation; source/quality changes preserve peers, while pause disables audio/video tracks together and keeps connections. Quality defaults to `maintain-resolution` with balanced/fluid options. - Production access is `screener-v2`: a valid private fragment grant enters directly; every code-only Viewer must first hold site access, after which public-watch accepts the code and private rooms still require their room password. `SITE_ACCESS_PASSWORD`, `/api/site-access`, and the site-access cookie are deployed atomically; raw credentials are never stored. - Web Hosts can set their local/session display name, and the current release advertises it through the existing presence wire. The name is not an account identity. -- Native v2 remains source-only (memory rooms, 300s reclaim); its Host now follows authenticated peer-assisted direct-child assignments with a two-edge cap and bounded SFU/selected-ingress failure, but production does not run it. +- Native v2 is source-only (memory rooms, 300s reclaim); it follows authenticated direct-child assignments up to two and fails unsupported SFU/selected ingress boundedly. - PR #44's STUN-only/SFU-UDP router and token-free fallback prewarm are process-enabled for all normal rooms: roots <=2, browser relay <=1, bounded failure. Room `1` is historical smoke; mobile/iPad viewers are leaves, healthy edges sticky, and HTTPS/WSS stays TLS/TCP. - After an answer, a generation-bound 15s initial-connect deadline enters ICE restart; success/replacement/disposal cancels it. It is deployed but not mobile-verified. - Room `1` can publish exactly `HIGH+LOW` with Dynacast/backup codec off and subscriber `HIGH` ceilings; it has no retained real media frame. @@ -30,10 +30,11 @@ Flagship; parallel; min run/smoke/rollback; benchmark later. Active UI/config/lo - Chrome 151 synthetic `1/3/5/8` and 720p30 quality-change runs kept 2/1 fanout and decoding; slowest first frame was 1.05 seconds and one relay close recovered in 5.32 seconds. This is control evidence only. - Host A+B and authenticated P2P Viewer C are sanitized, generation-bound, read-only, and fail closed for stale, ambiguous, or SFU-fed evidence; no raw media metadata is retained. - The `a11a73d` built-in TURN canary is rejected: local allocation passed, but direct Host/Pion Viewer failed before forced relay. Full rollback restored STUN-only client ICE and zero allocations. -- Chrome 151 loopbacks decoded VP8 299 and H.264 opt-in 299/298 rendered at 1280x720 without fatal/encoder errors. Hardware attribution, Pion timing, multi-viewer/endurance/public/native proof remain open; production stays VP8. -- Native Win11 audio is source-only/default-off. Target isolation was 4018x; one Viewer got 495 Opus packets. Package build passes; download, game sync, Win10, and other routes are open. +- Chrome 151 browser loopbacks decoded VP8 299 and H.264 opt-in 299/298 rendered at 1280x720 without fatal/encoder errors. Browser hardware attribution, multi-viewer/endurance and public proof remain open; production stays VP8. +- Native WGC/MF used no browser capture/encoder. One Chrome Viewer rendered 203 1280x720 frames and received 500 Opus packets; process/LUID-correlated `VideoEncode` was nonzero, with no fatal/encoder errors. +- Native Win11 audio is source-only/default-off: target isolation was 4018x and one Viewer got 495 Opus packets. Download, game sync, Win10, and other routes remain open. - Access protocol/config/HTTP/storage/SQLite/signaling focused tests pass, including commit-first teardown and v1 rollback. -- Native peer-assisted wire/session focused tests pass: authoritative direct children reconcile at no more than two, presence cannot create a third edge, stale revisions do not rebuild, and unsupported SFU/selected ingress fails once per current revision. +- Native wire/session tests enforce at most two authoritative children, no presence-created edge, stale-revision ignore, and one bounded unsupported-route failure per revision. - Source adds selected Host ingress and peer last-mile for every enabled room; focused routing/config tests pass with Peer ICE STUN-only. - Host names, the window-audio hint, all-room routing, and all-room Host selected ingress are active; the stale restart assertion is historical only. - The `fd76277` gate preserved the environment and SQLite hashes, v3/five rooms/room `1`, selected-edge/SFU/ordinary-ICE settings, services, and local/public health plus route/asset 200s. Site access returned the unauthenticated status and the retired endpoint returned 404; no TURN/SFU session ran. Exact `5e4a3679076a9ea2fe7a41fadf4be65a439db450` is the rollback release. @@ -55,7 +56,7 @@ Flagship; parallel; min run/smoke/rollback; benchmark later. Active UI/config/lo Repeat room `1` and fix only the identified SFU connect/source/publish/audio/transport layer; retain selected UDP, a rendered frame, edge caps and resource deltas. Recovery uses one ICE restart, one same-parent rebuild, one alternate peer, then SFU. Next gate exactly-two/Dynacast-off BWE on a zero-child SFU leaf, then root evacuation. -Package one-Viewer VP8/H.264/Opus first; WGC/MF hardware then Viewer2/FIFO are functional slices. Real-game A/V and resource benchmarks follow functional landing. +Native WGC/MF passed one-Viewer hardware; package download, Viewer2/FIFO and game A/V remain. Benchmark later. Ordinary Peer ICE stays STUN-only. Selected-edge Host ingress still needs one forced-relay canary before a media-success claim. ADR-0004 still needs the 30-minute `1/3/5/8` matrix; a separate 20-viewer gate must pass before changing the default eight. @@ -66,6 +67,5 @@ host/provider UDP 7882 and bounded services are deployed, but real external media/device evidence is absent. Shared IP remains smoke-only and cannot approve clean-port migration; the full gate still needs an isolated VM/IP. -- Whole-system versus selected-game audio for the first release. - Initial deployment region and network cohort. - Project license and distribution model. diff --git a/docs/方案设计.md b/docs/方案设计.md index e8696cb..40fa293 100644 --- a/docs/方案设计.md +++ b/docs/方案设计.md @@ -160,7 +160,9 @@ PoC 不强制 codec，也不做 SDP munging。先通过 stats 记录实际协商 `audio: true` 只是请求。若 `getAudioTracks()` 为空，分享者界面必须显示“当前来源没有可共享音频”。`systemAudio`、`windowAudio` 和 `audioSelection` 只影响选源器偏好，浏览器可以忽略且没有对应的标准音频来源类型读回，因此不能显示为系统、窗口或所选游戏音频保证。纯 Web 首版接受浏览器实际提供的标签页或系统音频，不承诺仅捕获某个游戏进程；系统音频可能把语音软件中观众自己的声音再次采入，这属于源内容泄漏而非 Host 页面回放，AEC 或 Web Audio 无法从已混合 track 恢复进程身份。Web 端必须保留 video-only 选择并明确风险，不能声称已隔离语音。 -当前 Windows Native sender 以所选游戏 PID 与进程创建时间为本地边界，使用 WASAPI process loopback 的 include-target-process-tree 捕获游戏及其子进程。目标进程无渲染流、退出或捕获失败时显示无声/未知，不回退为整机音频，也不上传 PID、路径、窗口标题、设备标识或原始音频。该 source-only 路径当前只支持 Windows 11；Windows 10 game-only 音频保持明确不支持，打包与其他媒体 route 仍待完成。 +当前 Windows Native sender 用本地 opaque ID 保存所选窗口的 HWND、PID 与进程创建时间；helper 在使用前重新验证三者绑定，并以 WASAPI process loopback 的 include-target-process-tree 捕获游戏及其子进程。目标进程无渲染流、退出或捕获失败时显示无声/未知，不回退为整机音频，也不上传 PID、路径、窗口标题、设备标识或原始音频。该路径只支持 Windows 11；Windows 10 game-only 音频保持明确不支持。 + +同一 helper 的显式 `native-window-h264` source 用 `CreateForWindow` 建立 WGC item，在所选 adapter 的 free-threaded D3D11 frame pool 接收 BGRA surface；D3D11 VideoProcessor 做固定 1280x720/30 aspect-fit NV12 转换，随后进入 adapter-LUID-bound、hardware-only asynchronous Media Foundation H.264 MFT。输出必须是 Annex-B `42c01f`，首次、周期和反馈恢复帧携带 SPS、PPS 与 IDR。PCM、H.264 access unit 和 local-only hardware status 通过一个有界 helper protocol 进入 generation-bound Go session；H.264 直接写现有 shared fanout，UI 只收一份单向预览副本，PCM 保持现有本地 WebCodecs Opus bridge。任一 Native capture/encode 失败都终止该 source，不静默改用软件 MFT、其他 codec、browser capture、monitor capture 或 system mix；browser VP8 保持默认，browser H.264 保持独立 opt-in。一名普通 Chrome Viewer 已 decoded/rendered 203 帧并收到 500 Opus packets；同一 helper PID 与 adapter LUID 的 `VideoEncode` 观测匹配。 直播中切换来源仍由明确的按钮手势重新调用 `getDisplayMedia()`，不改变房间、host generation 或媒体拓扑。选源取消或失败时继续发送旧流；成功后为每个现有 sender 调用 `replaceTrack()`，健康连接不重新协商。初始 offer 预留 send-only audio transceiver，使音频可以在 `null -> track -> null` 间切换。若某个 peer 的替换或回滚失败，只重建该 peer；新流提交后才停止旧流，并以当前 stream 身份保护旧 track 的迟到 `ended` 事件。统计码率累加器在换源后重置，避免把不同 track 的计数器混算。 diff --git a/docs/需求理解.md b/docs/需求理解.md index 6605ad5..18cae0d 100644 --- a/docs/需求理解.md +++ b/docs/需求理解.md @@ -53,7 +53,7 @@ - [ ] 分包/分层多树实验：把编码包、对象或可独立解码层拆到多个 peer 路径以分散上传，只能在独立 spike 中验证。必须保持亚秒级，处理多父重组、排序、丢包恢复、关键帧、同步和节点退出；未通过门槛前不进入产品路由控制器。 - [ ] SFU-root 中央兜底：产品部署目标必须提供经 canary 验收的单节点 SFU 资源；系统在 direct/peer UDP 不可用或明确拓扑预算无法接纳时自动把它作为主要中央兜底，不要求用户选模式。它只向总计一至两个必要 roots 出口；root 优先承担受限 peer descendants，没有可靠 relay root 时必要 viewer 只作为同一总上限内的零 descendants root。已关闭 PR #12 的显式整房 SFU 模式已被取代。room `1` 的历史 smoke 没有 retained SFU media；公网媒体、资源和恢复门禁仍适用于全站路由。 - [ ] 分享端桌面封装：用 Electron 固定 Chromium 版本、改善选源/热键/系统音频体验，同时保留 Web 观看端。 -- [ ] Windows 应用音频：Windows 11 原生候选默认仅捕获所选游戏及其子进程的音频，避免把语音聊天或通知声一并分享；“全部系统声音”只能由分享者显式选择。普通 Windows 10 不具备当前 process-loopback API 门槛，必须明确标为未解决/不支持而不是伪兼容。浏览器无法证明或隔离任意进程音频时必须如实提示，原生捕获失败、无声或进程退出不得静默扩大为整机捕获。 +- [ ] Windows 应用音频：Windows 11 Native sender 使用本地 opaque window target 绑定 HWND、PID 与进程创建时间，只捕获所选游戏及其子进程的音频，避免把语音聊天或通知声一并分享；“全部系统声音”只能由分享者显式选择。普通 Windows 10 不具备当前 process-loopback API 门槛，必须明确标为未解决/不支持而不是伪兼容。浏览器无法证明或隔离任意进程音频时必须如实提示，原生捕获失败、无声或进程退出不得静默扩大为整机捕获。 - [ ] 连接自检：在开始分享前验证信令、STUN 和 SFU ICE/UDP 可达性，并给出不暴露地址或凭据的诊断结果。启用 selected-edge TURN canary 时只报告脱敏的授权阶段与 selected candidate type，不输出 URL、username、credential、地址或原始 candidate；媒体 TCP 不在当前配置面。 - [ ] 受限网络增强：短期鉴权 TURN 只能由完整、默认关闭的部署 tuple 与当前 room controller 启用，并且只能在 direct/peer UDP 与 SFU/UDP 均已按有界梯级失败后，向 controller 选中的当前异常 edge 一次性授权。普通 peer PC 从创建起保持 STUN-only；不得向整个 participant、房间或所有 peer connections 预发 TURN candidates。凭据必须绑定当前授权 session 与 route/connection generation、短期有效、内存态、一次性消费且不进入 URL、日志、浏览器持久化或 SQLite。媒体 TCP 仍需独立 canary；两者都不能被描述为更高清的传输方式。 - [ ] 运行时编解码选择：先保留浏览器默认协商；只有端点能力相交、Media Capabilities 对目标分辨率/FPS/码率报告 supported/smooth/power-efficient，且实际 outbound codec、区间 encode time、游戏 FPS、CPU/GPU 与 sender 数量共同通过时，才调整 H.264、VP8、VP9 或 AV1 优先级。AV1 不得因压缩率推断而默认启用，任何候选都不得静默落入高负载软件编码。 @@ -61,7 +61,7 @@ ### P2 - 可选功能 -- [ ] 原生 Windows 捕获辅助程序，使用 WGC/DDA、WASAPI 和硬件编码器降低复制与编码开销。 +- [x] 原生 Windows 捕获辅助程序：显式 `native-window-h264` 模式使用同一 bound window target 的 WGC/D3D11 画面与 WASAPI process-tree PCM，并用 adapter-bound、hardware-only Media Foundation H.264 `42c01f` 直接进入现有 Go fanout；浏览器 VP8/H.264 是显式替代选项，VP8 默认，不允许运行时静默改用软件编码、显示器画面或系统混音。一名普通 Chrome Viewer 已 decoded/rendered 203 帧并收到 500 Opus packets，同一 helper PID/adapter 的 `VideoEncode` 与 LUID 匹配。 - [ ] iOS 原生分享端有界实验：当前不实现 ReplayKit/broadcast-extension 兼容栈。待 iOS 27 与对应 ScreenCaptureKit SDK 脱离 beta 并稳定后，只评估 ScreenCaptureKit system picker + fixed-`HIGH` + 一至两名 Web Viewer 的窄路径；VideoToolbox 必须要求并读回硬件编码，无法证明时 fail closed，不增加 ReplayKit fallback、`LOW`、AV1 或新拓扑。 - [ ] Viewer 向电视输出：系统级屏幕镜像现在可直接使用；产品只复用平台 AirPlay/Cast，不自造投屏协议。首个 Web 候选只在运行时检测 `HTMLMediaElement.remote`，并在真实 AirPlay/Cast 目标上证明当前 live WebRTC `srcObject` 可播放后才显示入口；不支持时保留浏览器原生媒体控件/系统镜像。投屏是 Viewer 的本地播放输出，不能改变 room、TURN/SFU/peer route、Host fanout 或服务端媒体分发。Google Cast receiver 需要可由接收器获取的媒体 URL，当前 P2P `MediaStream` 不满足该输入，因此不加入 Cast SDK/receiver。 - [ ] 多地区 SFU-root 调度，以及可选 TURN 兼容层的按地区、运营商可达率。 @@ -158,6 +158,7 @@ - Given 浏览器返回屏幕音轨，When 产品展示音频状态或未来连接详情，Then 只能把存在性、浏览器实际返回的 track settings、实际协商 codec/fmtp 和实际流量分别标为捕获、协商或观测事实；字段缺失必须显示为未知，且不得从 Opus 的 `48000/2` RTP 声明推断源采样率或实际 stereo。 - Given Web 或后续 native sender 分享电影/视频音频，When 进行产品验收，Then 必须用独立 A/B/C 证据关联 capture audio track/settings、实际 negotiated codec/fmtp、outbound/inbound 实际码率、loss、jitter、concealment、jitter-buffer 与 A/V sync；native sender 必须通过该音质和同步矩阵，且不得把 audio `maxBitrate` ceiling 当作提质证明。 - Given 分享者同时运行语音软件，When 选择仅分享游戏声音，Then supported Windows 11 native sender 只保留目标游戏及其子进程的渲染音频，观众、通知和其他应用声音不得进入分享；浏览器 system audio 无法满足时必须允许 video-only 或明确告知其可能包含语音/通知，不能用 AEC、降噪或来源标签伪称已隔离。 +- Given 分享者显式选择 Native window H.264 与一个本地窗口，When 分享开始，Then helper 必须在再次验证 HWND/PID/creation-time 后以 WGC/D3D11 和 hardware-only MF H.264 产生一份 `42c01f` Annex-B 输出并直接复用现有 Go/Pion fanout；Viewer 仍使用普通浏览器。硬件 MFT、WGC、音频或目标身份任一失败都明确停止该模式，不得静默改用软件编码、浏览器画面、显示器画面或系统混音。 ### 仓库可移植性 diff --git a/native/fixtures/mf-h264-hardware/README.md b/native/fixtures/mf-h264-hardware/README.md index d82505d..c138b3b 100644 --- a/native/fixtures/mf-h264-hardware/README.md +++ b/native/fixtures/mf-h264-hardware/README.md @@ -1,8 +1,10 @@ # Media Foundation H.264 Hardware Fixture -This is an offline Windows-only decision fixture. It is not linked into the -Screener sender and does not contact a room, signaling service, Viewer, or -production endpoint. +This is an offline Windows-only decision fixture. Its build compiles the same +Media Foundation encoder source used by `native/window-capture-helper`, under a +fixture-only macro for synthetic input and telemetry. The generated fixture is +not linked into the sender and does not contact a room, signaling service, +Viewer, or production endpoint. The fixture selects one DXGI adapter and one adapter-bound Media Foundation transform. It never enumerates a software MFT and has no codec or software @@ -36,10 +38,9 @@ The fixture prints bounded key/value evidence to stdout and never writes the encoded bitstream. A nonzero exit is a measured no-go at the reported stage; it must not be worked around by adding a software retry. -The Pion/two-binding and browser decode boundary remains separate. The current -machine has no Go toolchain in `PATH`, and the existing product bridge is still -no-go before first Go RTP. Coupling either boundary into this encoder fixture -would hide which layer failed. +Pion transport and browser decode remain separate from this offline fixture; +the product path reuses its exact `42c01f`, Annex-B, adapter-bound hardware +contract without adding those network concerns to this executable. Primary API references, accessed 2026-08-20: diff --git a/native/fixtures/mf-h264-hardware/build.ps1 b/native/fixtures/mf-h264-hardware/build.ps1 index 46883b3..9bb0864 100644 --- a/native/fixtures/mf-h264-hardware/build.ps1 +++ b/native/fixtures/mf-h264-hardware/build.ps1 @@ -28,7 +28,7 @@ if ([string]::IsNullOrWhiteSpace($installationPath)) { } $developerCommand = Join-Path $installationPath 'Common7\Tools\VsDevCmd.bat' -$sourcePath = Join-Path $fixtureDirectory 'main.cpp' +$sourcePath = Join-Path $fixtureDirectory '..\..\window-capture-helper\main.cpp' $executablePath = Join-Path $outputPath 'screener-mf-h264-fixture.exe' $objectPath = Join-Path $outputPath 'main.obj' @@ -36,7 +36,7 @@ New-Item -ItemType Directory -Path $outputPath -Force | Out-Null $compile = @( 'call "{0}" -arch=x64 -host_arch=x64 >nul' -f $developerCommand - 'cl.exe /nologo /std:c++20 /EHsc /W4 /WX /DUNICODE /D_UNICODE /DWIN32_LEAN_AND_MEAN /D_WIN32_WINNT=0x0A00 "{0}" /Fo:"{1}" /Fe:"{2}" /link mfplat.lib mf.lib mfuuid.lib d3d11.lib dxgi.lib dxguid.lib evr.lib ole32.lib oleaut32.lib pdh.lib' -f $sourcePath, $objectPath, $executablePath + 'cl.exe /nologo /std:c++20 /EHsc /W4 /WX /DSCREENER_H264_FIXTURE /DUNICODE /D_UNICODE /DWIN32_LEAN_AND_MEAN /D_WIN32_WINNT=0x0A00 "{0}" /Fo:"{1}" /Fe:"{2}" /link mfplat.lib mf.lib mfuuid.lib d3d11.lib dxgi.lib dxguid.lib evr.lib ole32.lib oleaut32.lib pdh.lib windowsapp.lib' -f $sourcePath, $objectPath, $executablePath ) -join ' && ' & cmd.exe /d /s /c $compile diff --git a/native/process-audio-helper/README.md b/native/process-audio-helper/README.md deleted file mode 100644 index 844c793..0000000 --- a/native/process-audio-helper/README.md +++ /dev/null @@ -1,22 +0,0 @@ -# Windows Process-Audio Helper - -This Windows 11 helper is the local input for Native sender process-tree audio. -It enumerates visible top-level windows locally and captures only the explicitly -selected process plus its children through WASAPI application loopback. It has -no whole-system fallback and no network code. - -The standard evaluation ZIP builds this helper beside the sender through -`scripts/package-native-sender.ps1`. For a helper-only development build, use -an output directory outside the repository (or set -`SCREENER_PROCESS_AUDIO_HELPER` for a development run): - -```powershell -$out = Join-Path ([IO.Path]::GetTempPath()) 'screener-native-audio' -./native/process-audio-helper/build.ps1 -OutputDirectory $out -``` - -The implementation follows Microsoft's MIT-licensed Application Loopback -sample contract without copying its WIL/Media Foundation fixture machinery: - -- -- diff --git a/native/sender/README.md b/native/sender/README.md index 92d8a16..ef59d8d 100644 --- a/native/sender/README.md +++ b/native/sender/README.md @@ -1,25 +1,27 @@ # Screener Native Sender for Windows x64 -This evaluation package contains the current Native sender and the Windows 11 -process-audio helper. It has no installer, updater, code signature, or bundled +This evaluation package contains the current Native sender and its Windows 11 +window-capture helper. It has no installer, updater, code signature, or bundled server. ## Run -1. Keep `screener-sender.exe` and `screener-process-audio.exe` in the same +1. Keep `screener-sender.exe` and `screener-window-capture.exe` in the same directory. 2. Run `screener-sender.exe`. It opens a loopback-only UI in the default browser; current Chrome or Edge is required. -3. Enter the Screener server and site access password, choose VP8 (default) - or experimental H.264, then choose the screen or window to share. -4. Process audio is off by default. To enable it, select the exact target in - the audio list and select the same target in the browser picker. +3. Enter the Screener server and site access password. Browser VP8 remains the + default; browser H.264 and Native window H.264 are explicit opt-ins. +4. For a browser video source, choose the browser screen/window and optionally + select the same window target for process-tree audio. Native window H.264 + requires one target and captures its video and process-tree audio together. 5. Close the console window to stop the sender. -Process audio requires Windows 11 build 22000 or newer. It captures only the -selected process tree and never falls back to the system mix. H.264 requests -browser hardware preference but does not prove which physical encoder Chrome -or Edge selected. Viewers continue using the normal Web client. +Native capture requires Windows 11 build 22000 or newer. It binds the selected +window to its process identity, never falls back to the system mix, and never +falls back from the hardware-only Media Foundation H.264 encoder to software. +Browser H.264 still requests browser hardware preference without proving which +physical encoder Chrome or Edge selected. Viewers use the normal Web client. Verify the downloaded ZIP against its adjacent `.sha256` file, then verify the files inside it with `SHA256SUMS.txt`. The package is unsigned, so Windows may diff --git a/native/sender/internal/app/server.go b/native/sender/internal/app/server.go index 63cde53..f179ff0 100644 --- a/native/sender/internal/app/server.go +++ b/native/sender/internal/app/server.go @@ -16,11 +16,18 @@ import ( "github.com/TNTcraftHIM/Screener/native/sender/internal/media" "github.com/TNTcraftHIM/Screener/native/sender/internal/remote" - "github.com/TNTcraftHIM/Screener/native/sender/internal/windowaudio" + "github.com/TNTcraftHIM/Screener/native/sender/internal/windowcapture" "github.com/coder/websocket" "github.com/coder/websocket/wsjson" ) +type videoSource string + +const ( + videoSourceBrowser videoSource = "browser" + videoSourceNativeWindow videoSource = "native-window-h264" +) + const ( processTokenHeader = "X-Screener-Process-Token" maxLocalJSONBytes = 8 << 10 @@ -38,43 +45,47 @@ type App struct { token string attachTimeout time.Duration - mu sync.Mutex - writeMu sync.Mutex - server *http.Server - listener net.Listener - expectedHost string - origin string - starting bool - generation uint64 - startCancel context.CancelFunc - startFailure string - session *remote.Session - sessionCancel context.CancelFunc - room remote.Room - codec media.Codec - mediaConn *websocket.Conn - mediaClaimed bool - mediaClaimID uint64 - nextClaimID uint64 - attachTimer *time.Timer - configTimeout time.Duration - activeViewers int - waitingViewers int - audioProvider windowaudio.Provider - audioTargets map[string]windowaudio.Target - audioTarget *windowaudio.Target + mu sync.Mutex + writeMu sync.Mutex + server *http.Server + listener net.Listener + expectedHost string + origin string + starting bool + generation uint64 + startCancel context.CancelFunc + startFailure string + session *remote.Session + sessionCancel context.CancelFunc + room remote.Room + codec media.Codec + mediaConn *websocket.Conn + mediaClaimed bool + mediaClaimID uint64 + nextClaimID uint64 + attachTimer *time.Timer + configTimeout time.Duration + activeViewers int + waitingViewers int + captureProvider windowcapture.Provider + windowTargets map[string]windowcapture.Target + windowTarget *windowcapture.Target + videoSource videoSource + keyFrameRequests chan struct{} } type startRequest struct { - ServerURL string `json:"serverUrl"` - Password string `json:"password"` - Codec string `json:"codec,omitempty"` - AudioTargetID string `json:"audioTargetId,omitempty"` + ServerURL string `json:"serverUrl"` + Password string `json:"password"` + Codec string `json:"codec,omitempty"` + VideoSource string `json:"videoSource,omitempty"` + WindowTargetID string `json:"windowTargetId,omitempty"` } type encoderConfig struct { Kind string `json:"kind"` Codec string `json:"codec"` + VideoSource string `json:"videoSource"` Width int `json:"width"` Height int `json:"height"` FPS int `json:"fps"` @@ -83,7 +94,7 @@ type encoderConfig struct { Audio bool `json:"audio"` } -type localAudioTarget struct { +type localWindowTarget struct { ID string `json:"id"` Title string `json:"title"` } @@ -96,11 +107,12 @@ func New() (*App, error) { ctx, cancel := context.WithCancel(context.Background()) return &App{ ctx: ctx, cancel: cancel, token: token, - attachTimeout: mediaAttachTimeout, - configTimeout: mediaConfigTimeout, - codec: media.CodecVP8, - audioProvider: windowaudio.NewProvider(), - audioTargets: make(map[string]windowaudio.Target), + attachTimeout: mediaAttachTimeout, + configTimeout: mediaConfigTimeout, + codec: media.CodecVP8, + videoSource: videoSourceBrowser, + captureProvider: windowcapture.NewProvider(), + windowTargets: make(map[string]windowcapture.Target), }, nil } @@ -157,8 +169,8 @@ func (app *App) ServeHTTP(response http.ResponseWriter, request *http.Request) { app.handleStart(response, request) case "/api/stop": app.handleStop(response, request) - case "/api/audio-targets": - app.handleAudioTargets(response, request) + case "/api/window-targets": + app.handleWindowTargets(response, request) case "/media": app.handleMedia(response, request) default: @@ -197,21 +209,35 @@ func (app *App) handleStart(response http.ResponseWriter, request *http.Request) writeJSON(response, http.StatusBadRequest, map[string]string{"error": "unsupported native codec"}) return } + source, validSource := parseVideoSource(strings.TrimSpace(input.VideoSource)) + if !validSource { + writeJSON(response, http.StatusBadRequest, map[string]string{"error": "unsupported video source"}) + return + } + if source == videoSourceNativeWindow && codec != media.CodecH264 { + writeJSON(response, http.StatusBadRequest, map[string]string{"error": "native window capture requires H.264"}) + return + } baseURL, err := normalizeRemoteBase(input.ServerURL) if err != nil { writeJSON(response, http.StatusBadRequest, map[string]string{"error": err.Error()}) return } app.mu.Lock() - var audioTarget *windowaudio.Target - if input.AudioTargetID != "" { - selected, ok := app.audioTargets[input.AudioTargetID] + var windowTarget *windowcapture.Target + if input.WindowTargetID != "" { + selected, ok := app.windowTargets[input.WindowTargetID] if !ok { app.mu.Unlock() - writeJSON(response, http.StatusBadRequest, map[string]string{"error": "selected audio target is no longer available"}) + writeJSON(response, http.StatusBadRequest, map[string]string{"error": "selected window target is no longer available"}) return } - audioTarget = &selected + windowTarget = &selected + } + if source == videoSourceNativeWindow && windowTarget == nil { + app.mu.Unlock() + writeJSON(response, http.StatusBadRequest, map[string]string{"error": "native window capture requires a selected window"}) + return } if app.starting || app.session != nil { app.mu.Unlock() @@ -227,11 +253,11 @@ func (app *App) handleStart(response http.ResponseWriter, request *http.Request) app.mu.Unlock() session, room, err := remote.Start(startContext, remote.StartOptions{ - BaseURL: baseURL, + BaseURL: baseURL, SiteAccessPassword: strings.TrimSpace(input.Password), - Codec: codec, - EnableAudio: audioTarget != nil, - OnEvent: func(event remote.Event) { app.emit(generation, event) }, + Codec: codec, + EnableAudio: windowTarget != nil, + OnEvent: func(event remote.Event) { app.emit(generation, event) }, }) app.mu.Lock() app.starting = false @@ -245,7 +271,8 @@ func (app *App) handleStart(response http.ResponseWriter, request *http.Request) app.sessionCancel = startCancel app.room = room app.codec = codec - app.audioTarget = audioTarget + app.videoSource = source + app.windowTarget = windowTarget app.attachTimer = time.AfterFunc(app.attachTimeout, func() { app.mu.Lock() expired := app.generation == generation && app.session == session && app.mediaConn == nil @@ -281,31 +308,31 @@ func (app *App) handleStop(response http.ResponseWriter, request *http.Request) response.WriteHeader(http.StatusNoContent) } -func (app *App) handleAudioTargets(response http.ResponseWriter, request *http.Request) { +func (app *App) handleWindowTargets(response http.ResponseWriter, request *http.Request) { if !app.authorizeAPI(response, request, http.MethodPost) { return } - targets, err := app.audioProvider.List(request.Context()) + targets, err := app.captureProvider.List(request.Context()) if err != nil { writeJSON(response, http.StatusOK, map[string]any{ - "supported": false, "targets": []localAudioTarget{}, - "reason": "Windows 11 process audio helper is unavailable", + "supported": false, "targets": []localWindowTarget{}, + "reason": "Windows 11 window capture helper is unavailable", }) return } - localTargets := make([]localAudioTarget, 0, len(targets)) - resolved := make(map[string]windowaudio.Target, len(targets)) + localTargets := make([]localWindowTarget, 0, len(targets)) + resolved := make(map[string]windowcapture.Target, len(targets)) for _, target := range targets { id, tokenErr := newOpaqueToken() if tokenErr != nil { - writeJSON(response, http.StatusInternalServerError, map[string]string{"error": "create local audio target identity failed"}) + writeJSON(response, http.StatusInternalServerError, map[string]string{"error": "create local window target identity failed"}) return } resolved[id] = target - localTargets = append(localTargets, localAudioTarget{ID: id, Title: target.Title}) + localTargets = append(localTargets, localWindowTarget{ID: id, Title: target.Title}) } app.mu.Lock() - app.audioTargets = resolved + app.windowTargets = resolved app.mu.Unlock() writeJSON(response, http.StatusOK, map[string]any{"supported": true, "targets": localTargets}) } @@ -388,8 +415,12 @@ func (app *App) handleMedia(response http.ResponseWriter, request *http.Request) return } var config encoderConfig + app.mu.Lock() + source := app.videoSource + target := app.windowTarget + app.mu.Unlock() if err = decodeLocalPayload(payload, &config); err != nil || - !validEncoderConfigForCodec(config, session.Codec()) || config.Audio != session.AudioEnabled() { + !validEncoderConfigForSource(config, session.Codec(), source) || config.Audio != session.AudioEnabled() { session.Fail(errors.New("local encoder configuration is invalid")) _ = connection.Close(websocket.StatusPolicyViolation, "invalid encoder config") return @@ -398,38 +429,80 @@ func (app *App) handleMedia(response http.ResponseWriter, request *http.Request) session.Fail(fmt.Errorf("acknowledge local encoder configuration: %w", err)) return } - app.mu.Lock() - target := app.audioTarget - app.mu.Unlock() captureContext, cancelCapture := context.WithCancel(app.ctx) - audioDone := make(chan struct{}) + captureDone := make(chan struct{}) + keyFrameRequests := make(chan struct{}, 1) + if source == videoSourceNativeWindow { + app.mu.Lock() + if app.generation == generation && app.session == session { + app.keyFrameRequests = keyFrameRequests + } + app.mu.Unlock() + } + writePCM := func(chunk windowcapture.PCMChunk) error { + envelope, encodeErr := media.EncodePacket(media.Packet{ + Kind: media.KindPCM, Timestamp100ns: chunk.Timestamp100ns, + Duration100ns: chunk.Duration100ns, Data: chunk.Data, + }) + if encodeErr != nil { + return encodeErr + } + return app.writeLocalBinary(connection, envelope) + } if target != nil { _ = app.writeLocal(connection, map[string]string{"kind": "audio-state", "state": "starting"}) - go func(selected windowaudio.Target) { - defer close(audioDone) - captureErr := app.audioProvider.Capture(captureContext, selected, func(chunk windowaudio.PCMChunk) error { - envelope, encodeErr := media.EncodePacket(media.Packet{ - Kind: media.KindPCM, Timestamp100ns: chunk.Timestamp100ns, - Duration100ns: chunk.Duration100ns, Data: chunk.Data, - }) - if encodeErr != nil { - return encodeErr - } - return app.writeLocalBinary(connection, envelope) - }) + go func(selected windowcapture.Target) { + defer close(captureDone) + var captureErr error + if source == videoSourceNativeWindow { + captureErr = app.captureProvider.CaptureWindow( + captureContext, selected, keyFrameRequests, writePCM, + func(accessUnit windowcapture.H264AccessUnit) error { + packet := media.Packet{ + Kind: media.KindVideo, KeyFrame: accessUnit.KeyFrame, + Timestamp100ns: accessUnit.Timestamp100ns, + Duration100ns: accessUnit.Duration100ns, Data: accessUnit.Data, + } + if writeErr := session.WriteMedia(packet); writeErr != nil { + return writeErr + } + envelope, encodeErr := media.EncodePacket(packet) + if encodeErr != nil { + return encodeErr + } + return app.writeLocalBinary(connection, envelope) + }, + func(status windowcapture.Status) error { + return app.writeLocal(connection, map[string]any{ + "kind": "native-video-state", "status": status, + }) + }, + ) + } else { + captureErr = app.captureProvider.CaptureAudio(captureContext, selected, writePCM) + } if captureErr != nil && captureContext.Err() == nil { - _ = app.writeLocal(connection, map[string]string{ - "kind": "audio-state", "state": "unavailable", - "message": "Target audio stopped; system audio was not substituted", - }) + if source == videoSourceNativeWindow { + session.Fail(errors.New("native window capture stopped")) + } else { + _ = app.writeLocal(connection, map[string]string{ + "kind": "audio-state", "state": "unavailable", + "message": "Target audio stopped; system audio was not substituted", + }) + } } }(*target) } else { - close(audioDone) + close(captureDone) } defer func() { + app.mu.Lock() + if app.generation == generation && app.keyFrameRequests == keyFrameRequests { + app.keyFrameRequests = nil + } + app.mu.Unlock() cancelCapture() - <-audioDone + <-captureDone }() for { messageType, payload, err = connection.Read(app.ctx) @@ -450,7 +523,9 @@ func (app *App) handleMedia(response http.ResponseWriter, request *http.Request) _ = connection.Close(websocket.StatusPolicyViolation, "invalid encoded media") return } - if packet.Kind != media.KindVideo && packet.Kind != media.KindOpus { + validKind := packet.Kind == media.KindOpus || + (source == videoSourceBrowser && packet.Kind == media.KindVideo) + if !validKind { session.Fail(errors.New("local media bridge received an invalid encoded media kind")) _ = connection.Close(websocket.StatusPolicyViolation, "encoded media required") return @@ -502,7 +577,14 @@ func (app *App) emit(generation uint64, event remote.Event) { } connection := app.mediaConn session := app.session + keyFrameRequests := app.keyFrameRequests app.mu.Unlock() + if event.Kind == "request-keyframe" && keyFrameRequests != nil { + select { + case keyFrameRequests <- struct{}{}: + default: + } + } if connection != nil { if err := app.writeLocal(connection, event); err != nil && event.Kind != "fatal" && session != nil { go session.Fail(fmt.Errorf("write local sender event: %w", err)) @@ -568,7 +650,9 @@ func (app *App) stopActiveGeneration(expectedGeneration uint64) { } app.room = remote.Room{} app.codec = media.CodecVP8 - app.audioTarget = nil + app.videoSource = videoSourceBrowser + app.windowTarget = nil + app.keyFrameRequests = nil app.activeViewers = 0 app.waitingViewers = 0 app.mu.Unlock() @@ -587,15 +671,30 @@ func (app *App) stopActiveGeneration(expectedGeneration uint64) { } func validEncoderConfig(config encoderConfig) bool { - return validEncoderConfigForCodec(config, media.CodecVP8) + return validEncoderConfigForSource(config, media.CodecVP8, videoSourceBrowser) } func validEncoderConfigForCodec(config encoderConfig, expected media.Codec) bool { + return validEncoderConfigForSource(config, expected, videoSourceBrowser) +} + +func validEncoderConfigForSource(config encoderConfig, expected media.Codec, source videoSource) bool { return config.Kind == "config" && config.Codec == string(expected) && + config.VideoSource == string(source) && config.Width == 1280 && config.Height == 720 && config.FPS == 30 && config.Bitrate == 3_000_000 && config.EncoderInstances == 1 } +func parseVideoSource(value string) (videoSource, bool) { + if value == "" || value == string(videoSourceBrowser) { + return videoSourceBrowser, true + } + if value == string(videoSourceNativeWindow) { + return videoSourceNativeWindow, true + } + return videoSourceBrowser, false +} + func decodeLocalJSON(request *http.Request, target any) error { payload, err := io.ReadAll(io.LimitReader(request.Body, maxLocalJSONBytes+1)) if err != nil || len(payload) > maxLocalJSONBytes { diff --git a/native/sender/internal/app/server_test.go b/native/sender/internal/app/server_test.go index 830625e..c464769 100644 --- a/native/sender/internal/app/server_test.go +++ b/native/sender/internal/app/server_test.go @@ -14,13 +14,13 @@ import ( "github.com/TNTcraftHIM/Screener/native/sender/internal/media" "github.com/TNTcraftHIM/Screener/native/sender/internal/remote" - "github.com/TNTcraftHIM/Screener/native/sender/internal/windowaudio" + "github.com/TNTcraftHIM/Screener/native/sender/internal/windowcapture" "github.com/coder/websocket" ) func TestEncoderConfigUsesTheFixedHighContract(t *testing.T) { config := encoderConfig{ - Kind: "config", Codec: "vp8", Width: 1280, Height: 720, + Kind: "config", Codec: "vp8", VideoSource: "browser", Width: 1280, Height: 720, FPS: 30, Bitrate: 3_000_000, EncoderInstances: 1, } if !validEncoderConfig(config) { @@ -34,7 +34,7 @@ func TestEncoderConfigUsesTheFixedHighContract(t *testing.T) { func TestEncoderConfigAcceptsExplicitH264OnlyForAnH264Session(t *testing.T) { config := encoderConfig{ - Kind: "config", Codec: "h264", Width: 1280, Height: 720, + Kind: "config", Codec: "h264", VideoSource: "browser", Width: 1280, Height: 720, FPS: 30, Bitrate: 3_000_000, EncoderInstances: 1, } if !validEncoderConfigForCodec(config, media.CodecH264) { @@ -45,14 +45,28 @@ func TestEncoderConfigAcceptsExplicitH264OnlyForAnH264Session(t *testing.T) { } } +func TestNativeWindowConfigRequiresExactH264Source(t *testing.T) { + config := encoderConfig{ + Kind: "config", Codec: "h264", VideoSource: "native-window-h264", + Width: 1280, Height: 720, FPS: 30, Bitrate: 3_000_000, EncoderInstances: 1, + } + if !validEncoderConfigForSource(config, media.CodecH264, videoSourceNativeWindow) { + t.Fatal("native window H.264 config was rejected") + } + if validEncoderConfigForSource(config, media.CodecH264, videoSourceBrowser) || + validEncoderConfigForSource(config, media.CodecVP8, videoSourceNativeWindow) { + t.Fatal("native window config crossed a source or codec boundary") + } +} + func TestDecodeLocalPayloadIsStrictAndSingleValued(t *testing.T) { - valid := []byte(`{"kind":"config","codec":"vp8","width":1280,"height":720,"fps":30,"bitrate":3000000,"encoderInstances":1}`) + valid := []byte(`{"kind":"config","codec":"vp8","videoSource":"browser","width":1280,"height":720,"fps":30,"bitrate":3000000,"encoderInstances":1}`) var config encoderConfig if err := decodeLocalPayload(valid, &config); err != nil { t.Fatal(err) } invalid := [][]byte{ - []byte(`{"kind":"config","codec":"vp8","width":1280,"height":720,"fps":30,"bitrate":3000000,"encoderInstances":1,"extra":true}`), + []byte(`{"kind":"config","codec":"vp8","videoSource":"browser","width":1280,"height":720,"fps":30,"bitrate":3000000,"encoderInstances":1,"extra":true}`), append(append([]byte(nil), valid...), valid...), } for _, payload := range invalid { @@ -62,20 +76,20 @@ func TestDecodeLocalPayloadIsStrictAndSingleValued(t *testing.T) { } } -func TestAudioTargetListingKeepsProcessIdentityBehindAnOpaqueLocalID(t *testing.T) { +func TestWindowTargetListingKeepsBoundIdentityBehindAnOpaqueLocalID(t *testing.T) { application, err := New() if err != nil { t.Fatal(err) } - application.audioProvider = fakeAudioProvider{targets: []windowaudio.Target{{ - PID: 424_242, CreationTime: 987_654_321, Title: "Game Window", + application.captureProvider = fakeCaptureProvider{targets: []windowcapture.Target{{ + WindowHandle: 123_456, PID: 424_242, CreationTime: 987_654_321, Title: "Game Window", }}} if _, err = application.Start(); err != nil { t.Fatal(err) } defer application.Close() - request, err := http.NewRequest(http.MethodPost, application.origin+"/api/audio-targets", nil) + request, err := http.NewRequest(http.MethodPost, application.origin+"/api/window-targets", nil) if err != nil { t.Fatal(err) } @@ -87,21 +101,21 @@ func TestAudioTargetListingKeepsProcessIdentityBehindAnOpaqueLocalID(t *testing. } defer response.Body.Close() var result struct { - Supported bool `json:"supported"` - Targets []localAudioTarget `json:"targets"` + Supported bool `json:"supported"` + Targets []localWindowTarget `json:"targets"` } if json.NewDecoder(response.Body).Decode(&result) != nil || !result.Supported || len(result.Targets) != 1 { - t.Fatalf("audio target response = %+v", result) + t.Fatalf("window target response = %+v", result) } if result.Targets[0].ID == "" || result.Targets[0].Title != "Game Window" || strings.Contains(result.Targets[0].ID, "424242") || strings.Contains(result.Targets[0].ID, "987654321") { - t.Fatalf("audio target was not locally anonymized: %+v", result.Targets[0]) + t.Fatalf("window target was not locally anonymized: %+v", result.Targets[0]) } application.mu.Lock() - resolved := application.audioTargets[result.Targets[0].ID] + resolved := application.windowTargets[result.Targets[0].ID] application.mu.Unlock() - if resolved.PID != 424_242 || resolved.CreationTime != 987_654_321 { - t.Fatal("opaque target did not resolve in process memory") + if resolved.WindowHandle != 123_456 || resolved.PID != 424_242 || resolved.CreationTime != 987_654_321 { + t.Fatal("opaque target did not resolve to its bound identity in process memory") } } @@ -472,7 +486,7 @@ func openConfiguredMedia(t *testing.T, application *App) *websocket.Conn { } t.Cleanup(func() { _ = connection.CloseNow() }) readLocalKind(t, connection, "ready") - config := []byte(`{"kind":"config","codec":"vp8","width":1280,"height":720,"fps":30,"bitrate":3000000,"encoderInstances":1}`) + config := []byte(`{"kind":"config","codec":"vp8","videoSource":"browser","width":1280,"height":720,"fps":30,"bitrate":3000000,"encoderInstances":1}`) writeContext, cancelWrite := context.WithTimeout(context.Background(), time.Second) err = connection.Write(writeContext, websocket.MessageText, config) cancelWrite() @@ -689,12 +703,23 @@ func appRoomResponse(origin string) map[string]any { } } -type fakeAudioProvider struct{ targets []windowaudio.Target } +type fakeCaptureProvider struct{ targets []windowcapture.Target } -func (provider fakeAudioProvider) List(context.Context) ([]windowaudio.Target, error) { +func (provider fakeCaptureProvider) List(context.Context) ([]windowcapture.Target, error) { return provider.targets, nil } -func (fakeAudioProvider) Capture(context.Context, windowaudio.Target, func(windowaudio.PCMChunk) error) error { +func (fakeCaptureProvider) CaptureAudio(context.Context, windowcapture.Target, func(windowcapture.PCMChunk) error) error { + return nil +} + +func (fakeCaptureProvider) CaptureWindow( + context.Context, + windowcapture.Target, + <-chan struct{}, + func(windowcapture.PCMChunk) error, + func(windowcapture.H264AccessUnit) error, + func(windowcapture.Status) error, +) error { return nil } diff --git a/native/sender/internal/app/ui/app.js b/native/sender/internal/app/ui/app.js index fa71672..9a3206a 100644 --- a/native/sender/internal/app/ui/app.js +++ b/native/sender/internal/app/ui/app.js @@ -25,11 +25,12 @@ const elements = { serverURL: document.querySelector("#server-url"), password: document.querySelector("#password"), codec: document.querySelector("#codec"), - audioTarget: document.querySelector("#audio-target"), - refreshAudio: document.querySelector("#refresh-audio"), + windowTarget: document.querySelector("#window-target"), + refreshWindows: document.querySelector("#refresh-windows"), start: document.querySelector("#start"), stop: document.querySelector("#stop"), preview: document.querySelector("#preview"), + nativePreview: document.querySelector("#native-preview"), empty: document.querySelector("#empty-state"), invite: document.querySelector("#invite"), inviteLink: document.querySelector("#invite-link"), @@ -48,6 +49,7 @@ const elements = { let stream = null; let processorReader = null; let encoder = null; +let videoDecoder = null; let audioEncoder = null; let silentAudioChunks = 0; let mediaSocket = null; @@ -59,10 +61,12 @@ let forceKeyFrame = true; let frameNumber = 0; let captureSettings = null; let nativeDiagnostics = null; +let nativeVideoStatus = null; let metrics = emptyMetrics(); let startGeneration = 0; let diagnosticsTimer = null; let pendingStartGeneration = 0; +let activeVideoSource = "browser"; if (!processToken) { showError("本地发送端令牌已失效，请重新启动 sender.exe"); @@ -80,47 +84,71 @@ elements.copy.addEventListener("click", async () => { showError("无法复制链接，请直接打开邀请链接"); } }); -elements.refreshAudio.addEventListener("click", () => void loadAudioTargets()); -if (processToken) void loadAudioTargets(); +elements.refreshWindows.addEventListener("click", () => void loadWindowTargets()); +if (processToken) void loadWindowTargets(); async function startSharing() { if (sharing || stopping || startInFlight) return; showError(""); - if (!("VideoEncoder" in window) || !("MediaStreamTrackProcessor" in window)) { + const videoSource = selectedVideoSource(); + const nativeWindow = videoSource === "native-window-h264"; + if (!nativeWindow && (!("VideoEncoder" in window) || !("MediaStreamTrackProcessor" in window))) { showError("此浏览器缺少 WebCodecs 屏幕编码支持，请使用当前版 Chrome 或 Edge"); return; } + if (nativeWindow && !("VideoDecoder" in window)) { + showError("此浏览器缺少原生 H.264 本地预览支持，请使用当前版 Chrome 或 Edge"); + return; + } startInFlight = true; elements.start.disabled = true; const generation = ++startGeneration; const codec = selectedCodec(); - const audioTargetId = elements.audioTarget.value; - if (audioTargetId && (!("AudioEncoder" in window) || !("AudioData" in window))) { + metrics.codec = codec; + const windowTargetId = elements.windowTarget.value; + activeVideoSource = videoSource; + if (nativeWindow && !windowTargetId) { + showError("原生 H.264 模式需要选择一个目标窗口"); + startInFlight = false; + elements.start.disabled = false; + return; + } + if (windowTargetId && (!("AudioEncoder" in window) || !("AudioData" in window))) { showError("当前浏览器缺少 WebCodecs Opus 编码支持，请关闭目标进程音频或使用当前版 Chrome / Edge"); startInFlight = false; elements.start.disabled = false; return; } - setStatus("选择屏幕"); try { - const capturedStream = await navigator.mediaDevices.getDisplayMedia({ - video: { - width: { ideal: WIDTH, max: WIDTH }, - height: { ideal: HEIGHT, max: HEIGHT }, - frameRate: { ideal: FPS, max: FPS }, - }, - audio: false, - }); - assertCurrentStart(generation, mediaSocket, capturedStream); - stream = capturedStream; - const videoTrack = stream.getVideoTracks()[0]; - if (!videoTrack) throw new Error("没有可用的视频轨道"); - await videoTrack.applyConstraints({ width: WIDTH, height: HEIGHT, frameRate: FPS }); - assertCurrentStart(generation); - captureSettings = videoTrack.getSettings(); + let videoTrack = null; + if (nativeWindow) { + await startNativeDecoder(generation); + captureSettings = { width: WIDTH, height: HEIGHT, frameRate: FPS }; + elements.preview.hidden = true; + elements.nativePreview.hidden = false; + } else { + setStatus("选择屏幕"); + const capturedStream = await navigator.mediaDevices.getDisplayMedia({ + video: { + width: { ideal: WIDTH, max: WIDTH }, + height: { ideal: HEIGHT, max: HEIGHT }, + frameRate: { ideal: FPS, max: FPS }, + }, + audio: false, + }); + assertCurrentStart(generation, mediaSocket, capturedStream); + stream = capturedStream; + videoTrack = stream.getVideoTracks()[0]; + if (!videoTrack) throw new Error("没有可用的视频轨道"); + await videoTrack.applyConstraints({ width: WIDTH, height: HEIGHT, frameRate: FPS }); + assertCurrentStart(generation); + captureSettings = videoTrack.getSettings(); + videoTrack.addEventListener("ended", () => void stopSharing(generation), { once: true }); + elements.preview.srcObject = stream; + elements.preview.hidden = false; + elements.nativePreview.hidden = true; + } updateDiagnostics(); - videoTrack.addEventListener("ended", () => void stopSharing(generation), { once: true }); - elements.preview.srcObject = stream; elements.empty.hidden = true; setStatus("创建房间"); @@ -129,7 +157,8 @@ async function startSharing() { serverUrl: elements.serverURL.value, password: elements.password.value, codec, - audioTargetId, + videoSource, + windowTargetId, }); assertCurrentStart(generation); if (pendingStartGeneration === generation) pendingStartGeneration = 0; @@ -139,15 +168,18 @@ async function startSharing() { elements.inviteLink.textContent = room.inviteUrl; elements.invite.hidden = false; - if (audioTargetId) await startAudioEncoder(generation); - await connectMediaSocket(generation, codec, Boolean(audioTargetId)); - assertCurrentStart(generation); - const activeEncoder = await startEncoder(generation, codec); + if (windowTargetId) await startAudioEncoder(generation); + await connectMediaSocket(generation, codec, videoSource, Boolean(windowTargetId)); assertCurrentStart(generation); + let activeEncoder = null; + if (!nativeWindow) { + activeEncoder = await startEncoder(generation, codec); + assertCurrentStart(generation); + } sharing = true; elements.stop.hidden = false; setStatus("正在分享"); - void readFrames(videoTrack, generation, activeEncoder, mediaSocket); + if (!nativeWindow) void readFrames(videoTrack, generation, activeEncoder, mediaSocket); } catch (error) { if (generation === startGeneration && !(error instanceof StaleStartError)) { showError(readableError(error, "无法开始分享")); @@ -159,7 +191,7 @@ async function startSharing() { } } -async function connectMediaSocket(generation, codec, audio) { +async function connectMediaSocket(generation, codec, videoSource, audio) { const url = new URL("/media", location.href); url.protocol = "ws:"; const socket = new WebSocket(url, [`screener.token.${processToken}`]); @@ -181,6 +213,7 @@ async function connectMediaSocket(generation, codec, audio) { socket.send(JSON.stringify({ kind: "config", codec, + videoSource, width: WIDTH, height: HEIGHT, fps: FPS, @@ -233,6 +266,34 @@ async function startEncoder(generation, codec) { return activeEncoder; } +async function startNativeDecoder(generation) { + const config = { + codec: "avc1.42c01f", + codedWidth: WIDTH, + codedHeight: HEIGHT, + optimizeForLatency: true, + }; + const support = await VideoDecoder.isConfigSupported(config); + assertCurrentStart(generation); + if (!support.supported) throw new Error("此浏览器不支持 H.264 42c01f 本地预览"); + let activeDecoder; + activeDecoder = new VideoDecoder({ + output: (frame) => { + try { + if (generation === startGeneration && videoDecoder === activeDecoder) { + const context = elements.nativePreview.getContext("2d"); + context?.drawImage(frame, 0, 0, WIDTH, HEIGHT); + } + } finally { + frame.close(); + } + }, + error: (error) => void failSharing(generation, `原生 H.264 本地预览失败：${error.message}`), + }); + videoDecoder = activeDecoder; + activeDecoder.configure(support.config); +} + async function readFrames(videoTrack, generation, activeEncoder, activeSocket) { const processor = new MediaStreamTrackProcessor({ track: videoTrack }); const reader = processor.readable.getReader(); @@ -319,7 +380,14 @@ function sendEncodedChunk(generation, activeEncoder, chunk) { function handleLocalEvent(generation, socket, event) { if (generation !== startGeneration || socket !== mediaSocket) return; if (event.data instanceof ArrayBuffer) { - handlePCM(generation, event.data); + const kind = event.data.byteLength > 1 ? new DataView(event.data).getUint8(1) : 0; + if (kind === MEDIA_KIND_PCM) { + handlePCM(generation, event.data); + } else if (kind === MEDIA_KIND_VIDEO && activeVideoSource === "native-window-h264") { + handleNativeVideo(generation, event.data); + } else { + void failSharing(generation, "本地媒体桥接返回了意外的二进制数据"); + } return; } if (typeof event.data !== "string") return; @@ -339,11 +407,40 @@ function handleLocalEvent(generation, socket, event) { ? "不可用（未回退系统声音）" : "正在连接目标进程"; } + if (message.kind === "native-video-state") { + nativeVideoStatus = { ...nativeVideoStatus, ...message.status }; + updateDiagnostics(); + setStatus(message.status?.state === "active" ? "正在分享（原生 H.264）" : "正在启动原生硬件采集"); + } if (message.kind === "fatal") { void failSharing(generation, message.message || "分享会话已失败并停止"); } } +function handleNativeVideo(generation, message) { + if (generation !== startGeneration || !videoDecoder || videoDecoder.state !== "configured") return; + try { + const view = new DataView(message); + const flags = view.getUint8(2); + if (message.byteLength <= ENVELOPE_HEADER_BYTES || view.getUint8(0) !== 1 || + view.getUint8(1) !== MEDIA_KIND_VIDEO || flags > 1 || view.getUint8(3) !== 0) { + throw new Error("本地 H.264 envelope 无效"); + } + const data = new Uint8Array(message, ENVELOPE_HEADER_BYTES); + if (data.byteLength < 1 || data.byteLength > MAX_FRAME_BYTES) { + throw new Error("本地 H.264 access unit 长度无效"); + } + videoDecoder.decode(new EncodedVideoChunk({ + type: flags === 1 ? "key" : "delta", + timestamp: Number(view.getBigUint64(4, false) / 10n), + duration: Number(view.getBigUint64(12, false) / 10n), + data, + })); + } catch (error) { + void failSharing(generation, readableError(error, "处理本地 H.264 预览失败")); + } +} + async function failSharing(generation, message) { if (generation !== startGeneration || stopping) return; showError(message); @@ -381,6 +478,15 @@ async function stopSharing(expectedGeneration = startGeneration) { rememberError(error); } encoder = null; + try { + if (videoDecoder && videoDecoder.state !== "closed") videoDecoder.close(); + } catch (error) { + rememberError(error); + } + videoDecoder = null; + elements.nativePreview.getContext("2d")?.clearRect(0, 0, WIDTH, HEIGHT); + elements.nativePreview.hidden = true; + elements.preview.hidden = false; try { if (audioEncoder && audioEncoder.state !== "closed") audioEncoder.close(); } catch (error) { @@ -408,6 +514,8 @@ async function stopSharing(expectedGeneration = startGeneration) { silentAudioChunks = 0; captureSettings = null; nativeDiagnostics = null; + nativeVideoStatus = null; + activeVideoSource = "browser"; metrics = emptyMetrics(); if (diagnosticsTimer !== null) { clearTimeout(diagnosticsTimer); @@ -488,25 +596,25 @@ async function localRequest(path, body) { return result; } -async function loadAudioTargets() { - elements.refreshAudio.disabled = true; - const previous = elements.audioTarget.value; +async function loadWindowTargets() { + elements.refreshWindows.disabled = true; + const previous = elements.windowTarget.value; try { - const result = await localRequest("/api/audio-targets"); - elements.audioTarget.replaceChildren(new Option("关闭", "")); + const result = await localRequest("/api/window-targets"); + elements.windowTarget.replaceChildren(new Option("关闭窗口音频", "")); for (const target of Array.isArray(result.targets) ? result.targets : []) { - if (target?.id && target?.title) elements.audioTarget.add(new Option(target.title, target.id)); + if (target?.id && target?.title) elements.windowTarget.add(new Option(target.title, target.id)); } - if ([...elements.audioTarget.options].some((option) => option.value === previous)) { - elements.audioTarget.value = previous; + if ([...elements.windowTarget.options].some((option) => option.value === previous)) { + elements.windowTarget.value = previous; } - elements.audioTarget.disabled = !result.supported; + elements.windowTarget.disabled = !result.supported; elements.audioStatus.textContent = result.supported ? "关闭" : "Windows 11 helper 不可用"; } catch (error) { - elements.audioTarget.disabled = true; + elements.windowTarget.disabled = true; elements.audioStatus.textContent = readableError(error, "窗口列表不可用"); } finally { - elements.refreshAudio.disabled = false; + elements.refreshWindows.disabled = false; } } @@ -595,7 +703,16 @@ function updateDiagnostics() { const height = captureSettings?.height ?? "未知"; const rate = captureSettings?.frameRate ?? "未知"; elements.captureDiagnostic.textContent = `${width} × ${height} · ${rate} FPS`; - elements.encoderDiagnostic.textContent = `${codecLabel(metrics.codec)} · 3 Mbps 上限 · 单对象 · 硬件偏好 ${hardwarePreferenceLabel(metrics.hardwarePreference)} · 硬件证据 ${hardwareEvidenceLabel(metrics.hardwareEvidence)}`; + if (activeVideoSource === "native-window-h264") { + const mftState = nativeVideoStatus?.state === "active" + ? "hardware-only MFT 已绑定并输出" + : nativeVideoStatus?.state === "starting" + ? "hardware-only MFT 已绑定，等待首帧" + : "hardware-only MFT 等待状态"; + elements.encoderDiagnostic.textContent = `H.264 · 3 Mbps 上限 · 单 MF 编码器 · ${mftState}`; + } else { + elements.encoderDiagnostic.textContent = `${codecLabel(metrics.codec)} · 3 Mbps 上限 · 单对象 · 硬件偏好 ${hardwarePreferenceLabel(metrics.hardwarePreference)} · 硬件证据 ${hardwareEvidenceLabel(metrics.hardwareEvidence)}`; + } elements.bridgeDiagnostic.textContent = `读取 ${metrics.framesRead} · 提交 ${metrics.framesSubmitted} · 输出 ${metrics.encoderOutputs} · 编码队列丢弃 ${metrics.encoderQueueDrops} · 编码队列峰值 ${metrics.encoderQueuePeak} 帧 · socket 峰值 ${metrics.socketBufferedPeak} B`; const helper = nativeDiagnostics?.media; const queue = helper?.queue; @@ -645,7 +762,11 @@ function boundedCount(value) { } function selectedCodec() { - return elements.codec?.value === "h264" ? "h264" : "vp8"; + return elements.codec?.value === "h264" || elements.codec?.value === "native-h264" ? "h264" : "vp8"; +} + +function selectedVideoSource() { + return elements.codec?.value === "native-h264" ? "native-window-h264" : "browser"; } function emptyMetrics() { diff --git a/native/sender/internal/app/ui/index.html b/native/sender/internal/app/ui/index.html index 2a75ffa..5191958 100644 --- a/native/sender/internal/app/ui/index.html +++ b/native/sender/internal/app/ui/index.html @@ -22,6 +22,7 @@
+
选择屏幕后，邀请链接会显示在这里