From 6cd99545e662fca49eac0758ec29d4b58c9d357c Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Fri, 21 Aug 2026 02:44:46 +0800 Subject: [PATCH] feat(native): add window-scoped process audio (#93) Add opt-in Windows 11 target-process audio capture, one AudioEncoder Opus stream, and one shared Pion audio track beside native video. --- docs/adr/0008-window-scoped-audio-capture.md | 36 +- docs/project-memory.md | 2 +- docs/research/browser-screen-audio-quality.md | 18 + docs/status.md | 4 +- native/process-audio-helper/README.md | 20 ++ native/process-audio-helper/build.ps1 | 40 +++ native/process-audio-helper/main.cpp | 334 ++++++++++++++++++ native/sender/go.mod | 2 +- native/sender/internal/app/server.go | 122 ++++++- native/sender/internal/app/server_test.go | 78 +++- native/sender/internal/app/ui/app.js | 149 +++++++- native/sender/internal/app/ui/index.html | 11 +- native/sender/internal/app/ui/styles.css | 8 +- native/sender/internal/media/audio.go | 178 ++++++++++ native/sender/internal/media/audio_test.go | 56 +++ native/sender/internal/media/envelope.go | 104 ++++++ native/sender/internal/media/fanout.go | 32 +- native/sender/internal/media/fanout_test.go | 22 +- native/sender/internal/remote/peer.go | 27 +- native/sender/internal/remote/session.go | 62 +++- .../sender/internal/windowaudio/provider.go | 25 ++ .../internal/windowaudio/provider_other.go | 15 + .../internal/windowaudio/provider_windows.go | 145 ++++++++ .../windowaudio/provider_windows_test.go | 100 ++++++ scripts/native-one-viewer-gate.ts | 35 +- 25 files changed, 1516 insertions(+), 109 deletions(-) create mode 100644 native/process-audio-helper/README.md create mode 100644 native/process-audio-helper/build.ps1 create mode 100644 native/process-audio-helper/main.cpp create mode 100644 native/sender/internal/media/audio.go create mode 100644 native/sender/internal/media/audio_test.go create mode 100644 native/sender/internal/media/envelope.go create mode 100644 native/sender/internal/windowaudio/provider.go create mode 100644 native/sender/internal/windowaudio/provider_other.go create mode 100644 native/sender/internal/windowaudio/provider_windows.go create mode 100644 native/sender/internal/windowaudio/provider_windows_test.go diff --git a/docs/adr/0008-window-scoped-audio-capture.md b/docs/adr/0008-window-scoped-audio-capture.md index 51396f3..229da1b 100644 --- a/docs/adr/0008-window-scoped-audio-capture.md +++ b/docs/adr/0008-window-scoped-audio-capture.md @@ -2,8 +2,8 @@ Date: 2026-08-21 -Status: Accepted for the browser hint; native Windows implementation is a -planned P1 input. +Status: Accepted. The browser hint and opt-in Native Windows P1 slice are +source-complete; Native remains unpackaged, default-off, and undeployed. ## Context @@ -35,8 +35,9 @@ Windows 11 desktop (build 22000 or newer) and is not a Windows 10 fallback. 3. The native sender's next audio slice is an explicit opt-in `window-process-audio` mode on Windows 11. The local capture session owns one selected target PID/process tree, one WGC/DXGI video source, and one - WASAPI loopback audio source. Process IDs, titles, paths, device identity, - and PCM stay local; they do not enter signaling, URLs, logs, or storage. + WASAPI loopback audio source. Process IDs, creation tokens, titles, paths, + device identity, and PCM stay local; they do not enter signaling, URLs, + logs, or storage. 4. If the target process exits, produces no render stream, or the loopback activation is denied, the sender reports `audio unavailable` or `audio silent` and stops/asks the user. It never widens to whole-system audio. @@ -48,7 +49,7 @@ Windows 11 desktop (build 22000 or newer) and is not a Windows 10 fallback. This is an interface boundary, not a new framework or a current wire change: ```text -NativeCaptureTarget { pid: uint32, includeProcessTree: true } +NativeCaptureTarget { pid: uint32, creationTime: uint64, includeProcessTree: true } NativeCaptureSession.start(target) -> VideoFrames(WGC/DXGI) + AudioPcm(WASAPI process loopback) -> existing native sender timeline/encoder/fanout @@ -66,7 +67,8 @@ The first product-wiring slice is deliberately smaller than the complete native capture boundary. It adds an explicit Windows 11 `window-process-audio` input while leaving the current Web video capture and codec path unchanged. A small Windows helper resolves the selected target PID -locally and sends timestamped PCM through the authenticated loopback bridge. +and creation time locally, revalidates both immediately before activation, and +sends timestamped PCM through the authenticated loopback bridge. The existing Chrome/Edge sender UI uses `AudioEncoder` with `codec: "opus"`, 48 kHz, two channels, and 20 ms frames, then returns the encoded Opus packets to the existing Go sender. This reuses the current browser prerequisite and avoids @@ -84,8 +86,8 @@ QPC origin when native video replaces the Web source. Expected implementation size is about **500--800 new LOC**, excluding the browser/Windows SDK and existing video/MF fixture code: Windows loopback capture and local IPC (250--400), Go audio envelope/timeline/track (150--250), -and UI lifecycle/status plus focused tests (100--150). This is a planned P1 -input, not current product behavior or a deployment switch. +and UI lifecycle/status plus focused tests (100--150). The landed slice keeps +that boundary and is not a deployment switch. The two source clocks are both converted from 100-ns QPC: WGC `SystemRelativeTime` and WASAPI `IAudioCaptureClient::GetBuffer`'s @@ -101,7 +103,8 @@ or `AUDCLNT_E_SERVICE_NOT_RUNNING`, protected-content silence, and a missing render stream produce `audio unavailable` or `audio silent`. There is no retry to whole-system loopback. No administrator privilege is required, but the helper must run in the interactive user's session and honor Windows consent and -privacy settings. PID, title, path, device identity, and PCM remain local. +privacy settings. PID, creation token, title, path, device identity, and PCM +remain local. The first smoke is one direct Viewer: a test window emits a known tone and visual marker, while an independent voice process and notification emit @@ -109,7 +112,9 @@ different markers. Require an Opus inbound track and rendered video/audio, monotonic QPC timestamps, and a bounded A/V offset; verify the unrelated markers are absent and that target exit/silence stops or asks without widening capture. SFU/TURN, second Viewer, endurance, and the native WGC video swap are -follow-up gates. +follow-up gates. On 2026-08-21, 100 20-ms chunks isolated a 440 Hz target from +an independent 880 Hz process by 4017.8x. Chrome 151 then received one audio +track and 495 inbound Opus packets while video decoded/rendered 296 frames. ## Consequences @@ -122,10 +127,7 @@ follow-up gates. ## Follow-Up TODO -- Add the Windows-only WGC + WASAPI process-loopback fixture beside the existing - Media Foundation H.264 fixture, then land the bounded audio-only P1 slice. -- Feed its timestamped PCM/video pair into the existing native sender only - after the isolation and sync checks pass; native WGC video replacement stays - a separate follow-up. -- Retain a short matrix result in `docs/research/`; do not enable the mode by - default or deploy it from this ADR alone. +- Package the helper beside the sender and keep explicit target selection + default-off. +- Keep native WGC video, a second Viewer, SFU/TURN, endurance, and real-game + A/V sync as separate follow-ups. diff --git a/docs/project-memory.md b/docs/project-memory.md index a06e0c7..4cffff9 100644 --- a/docs/project-memory.md +++ b/docs/project-memory.md @@ -30,7 +30,7 @@ Last updated: 2026-08-21 - C+B quality reparenting is edge-local and cooldown-bound; no score, timer, or global parent penalty. - Flagship media is UDP; HTTPS/WSS stays TLS/TCP; the old release remains rollback-only. - Treat settings as ceilings and degradation as unclassified. Use correlated Host A+B/Viewer C and one-variable evidence; never force AV1, infer by UA, or create a composite score. -- System capture can cause film-audio loss/self-echo; Web cannot isolate processes. Diagnose A/B/C and sync; Windows 11 native game-tree audio is a candidate, Windows 10 unresolved. See `docs/research/browser-screen-audio-quality.md`. +- Web capture cannot isolate process audio. ADR-0008 Windows 11 native audio is opt-in/source-only: opaque local selection, no system-mix fallback or exposed PID, and one shared Opus track; one direct Viewer received 495 packets. Packaging, game A/V sync, other routes, and Windows 10 remain open. - Viewer-local names and the opt-in Web participant roster remain control-plane-only. Web Host names are deployed in the current release and stay socket/localStorage-only; Native remains outside the capability boundary. - Do not add scene detection, dynamic-FPS control, or forced AV1 without negotiation, encode, game, CPU/GPU, and sender evidence. - ADR-0002 grant/public access, the v3 room-password migration, and Web Host display names are deployed. Names/presence remain session-only without account or roster tables. diff --git a/docs/research/browser-screen-audio-quality.md b/docs/research/browser-screen-audio-quality.md index 9345257..ec1ae1f 100644 --- a/docs/research/browser-screen-audio-quality.md +++ b/docs/research/browser-screen-audio-quality.md @@ -208,6 +208,24 @@ known target tone/visual marker and independent voice/notification markers; SFU/TURN, a second Viewer, endurance, and native WGC video replacement remain later gates. +### Retained P1 Result (2026-08-21) + +The source-complete Windows helper emits process-tree-only 48 kHz stereo s16 in +20-ms chunks. Chrome 151 encodes the chunks once with `AudioEncoder`/Opus, and +Go packetizes the same encoded stream once into a shared Pion track. The UI +receives an opaque memory-only target ID. PID and the enumeration-time process +creation token are revalidated locally and never enter public signaling, a URL, +storage, or diagnostics. + +One bounded run played 440 Hz in the selected target and 880 Hz in an +independent process. Across 100 chunks, amplitudes were 0.240225 and 0.0000598 +(4017.8x); no system-mix fallback was used. One direct Viewer then received one +audio track and 495 inbound Opus packets while video decoded/rendered 296 +frames. There was no fatal or encoder error. The sole failed assertion was the +existing generic Pion two-second outbound snapshot, which also missed fresh +deltas in earlier video loopbacks. This is functional evidence, not packaging, +real-game sync, second-Viewer, SFU/TURN, or endurance evidence. + Run one bounded matrix rather than a full route Cartesian product: exact production and current `main` on Windows Chrome/Edge for tab/window/monitor, audio selected/unselected, and a simultaneous voice call; then the native diff --git a/docs/status.md b/docs/status.md index 278c9f6..1a9fc71 100644 --- a/docs/status.md +++ b/docs/status.md @@ -42,6 +42,7 @@ Flagship; parallel; min run/smoke/rollback; benchmark later. Active UI/config/lo - Host A+B and authenticated P2P Viewer C are sanitized, generation-bound, read-only, and fail closed for stale, ambiguous, or SFU-fed evidence; no raw media metadata is retained. - The `a11a73d` built-in TURN canary is rejected: local allocation passed, but direct Host/Pion Viewer failed before forced relay. Full rollback restored STUN-only client ICE and zero allocations. - Chrome 151 loopbacks decoded VP8 299 and H.264 opt-in 299/298 rendered at 1280x720 without fatal/encoder errors. Hardware attribution, Pion timing, multi-viewer/endurance/public/native proof remain open; production stays VP8. +- Native Win11 audio is source-only/default-off. Smoke isolated target 4018x; one Viewer got 495 Opus packets. Packaging, game sync, Win10, and other routes remain open. - Access protocol/config/HTTP/storage/SQLite/signaling focused tests pass, including commit-first teardown and v1 rollback. - Source adds selected Host ingress and peer last-mile for every enabled room; focused routing/config tests pass with Peer ICE STUN-only. - `61a87ae` Host names, `d1a4fd4` window-audio hint, and `cf149df` all-room routing are active; the stale restart assertion is historical only. @@ -52,7 +53,6 @@ Flagship; parallel; min run/smoke/rollback; benchmark later. Active UI/config/lo - The final deployment proves process/config health, not a retained SFU frame, retry/failback, selected UDP pair, admission rescue, edge cap, quality, or resource deltas; game-share load/blur remains unclassified. - Android Chrome/iOS Safari Viewer leaves remain unverified and conservatively leaf-only. Mobile Web Host is unsupported; native senders are planned only. - Silent partitions can wait 30 to 60 seconds for heartbeat detection before the default 5-second grace; this remains unverified. -- Real audio and heterogeneous clients remain unverified. System capture can lose film audio or echo voice calls; Web has no process isolation. Diagnose A/B/C and sync; gate Windows 11 game audio; Windows 10 unresolved. - The former release reportedly sustained bandwidth-limited blur on a capable LAN; Host refresh recovered while Viewer refresh did not. The new deployment has not yet reproduced or cleared it. - ADR-0006 has one Viewer proof; two-edge/FIFO, physical hardware, endurance, public, packaged-native, and browser-diversity proof remain open. - Browser fanout is host two/viewer one; any accepted endpoint relay stays capped at two downstream edges. @@ -74,7 +74,7 @@ evacuation. Native source accepts increasing capture timestamps shorter than duration; VP8 and explicit H.264 Viewer decode/render are recorded. Hardware preference is not physical proof. Next prove fresh Pion outbound diagnostics before viewer 2/FIFO/hardware/endurance/deploy. -Audio A/B/C may proceed without displacing P0. +Package Native audio and test one real-game A/V pair later without displacing P0. Ordinary Peer ICE stays STUN-only; participant-wide TURN is removed. The selected-edge tuple is configured in production, but Host ingress still needs one forced-relay canary before any media-success claim. Retained performance/resource diff --git a/native/process-audio-helper/README.md b/native/process-audio-helper/README.md new file mode 100644 index 0000000..d843bce --- /dev/null +++ b/native/process-audio-helper/README.md @@ -0,0 +1,20 @@ +# Windows Process-Audio Helper + +This Windows 11 helper is the local input for Native sender process-tree audio. +It enumerates visible top-level windows locally and captures only the explicitly +selected process plus its children through WASAPI application loopback. It has +no whole-system fallback and no network code. + +Build it outside the repository, then place both executables in the same output +directory (or set `SCREENER_PROCESS_AUDIO_HELPER` for a development run): + +```powershell +$out = Join-Path ([IO.Path]::GetTempPath()) 'screener-native-audio' +./native/process-audio-helper/build.ps1 -OutputDirectory $out +``` + +The implementation follows Microsoft's MIT-licensed Application Loopback +sample contract without copying its WIL/Media Foundation fixture machinery: + +- +- diff --git a/native/process-audio-helper/build.ps1 b/native/process-audio-helper/build.ps1 new file mode 100644 index 0000000..59e14ca --- /dev/null +++ b/native/process-audio-helper/build.ps1 @@ -0,0 +1,40 @@ +[CmdletBinding()] +param( + [Parameter(Mandatory = $true)] + [string]$OutputDirectory +) + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' + +$helperDirectory = Split-Path -Parent $MyInvocation.MyCommand.Path +$repositoryRoot = [IO.Path]::GetFullPath((Join-Path $helperDirectory '..\..')) +$outputPath = [IO.Path]::GetFullPath($OutputDirectory) +$repositoryPrefix = $repositoryRoot.TrimEnd([IO.Path]::DirectorySeparatorChar) + [IO.Path]::DirectorySeparatorChar +if ($outputPath.Equals($repositoryRoot, [StringComparison]::OrdinalIgnoreCase) -or + $outputPath.StartsWith($repositoryPrefix, [StringComparison]::OrdinalIgnoreCase)) { + throw 'OutputDirectory must be outside the repository.' +} + +$vswhere = Join-Path ${env:ProgramFiles(x86)} 'Microsoft Visual Studio\Installer\vswhere.exe' +$installationPath = & $vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath +if ([string]::IsNullOrWhiteSpace($installationPath)) { + throw 'Visual Studio C++ Build Tools were not found.' +} + +$developerCommand = Join-Path $installationPath 'Common7\Tools\VsDevCmd.bat' +$sourcePath = Join-Path $helperDirectory 'main.cpp' +$executablePath = Join-Path $outputPath 'screener-process-audio.exe' +$objectPath = Join-Path $outputPath 'main.obj' +New-Item -ItemType Directory -Path $outputPath -Force | Out-Null + +$compile = @( + 'call "{0}" -arch=x64 -host_arch=x64 >nul' -f $developerCommand + 'cl.exe /nologo /std:c++20 /EHsc /W4 /WX /DUNICODE /D_UNICODE /DWIN32_LEAN_AND_MEAN /D_WIN32_WINNT=0x0A00 /DNTDDI_VERSION=0x0A00000A "{0}" /Fo:"{1}" /Fe:"{2}" /link ole32.lib mmdevapi.lib runtimeobject.lib user32.lib' -f $sourcePath, $objectPath, $executablePath +) -join ' && ' + +& cmd.exe /d /s /c $compile +if ($LASTEXITCODE -ne 0) { + throw "Process-audio helper compilation failed with exit code $LASTEXITCODE." +} +Write-Output $executablePath diff --git a/native/process-audio-helper/main.cpp b/native/process-audio-helper/main.cpp new file mode 100644 index 0000000..95c5b6e --- /dev/null +++ b/native/process-audio-helper/main.cpp @@ -0,0 +1,334 @@ +#define NOMINMAX +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +using Microsoft::WRL::ComPtr; +using Microsoft::WRL::FtmBase; +using Microsoft::WRL::Make; +using Microsoft::WRL::RuntimeClass; +using Microsoft::WRL::RuntimeClassFlags; +using Microsoft::WRL::ClassicCom; + +namespace { +constexpr UINT32 kSampleRate = 48'000; +constexpr UINT16 kChannels = 2; +constexpr UINT16 kBytesPerSample = 2; +constexpr UINT32 kFramesPerChunk = 960; +constexpr DWORD kBytesPerChunk = kFramesPerChunk * kChannels * kBytesPerSample; +constexpr UINT64 kChunkDuration100ns = 200'000; +constexpr size_t kMaxWindows = 100; + +class ActivationHandler final : public RuntimeClass< + RuntimeClassFlags, FtmBase, IActivateAudioInterfaceCompletionHandler> { + public: + explicit ActivationHandler(HANDLE completed) : completed_(completed) {} + + STDMETHODIMP ActivateCompleted(IActivateAudioInterfaceAsyncOperation* operation) override { + ComPtr activated; + HRESULT activationResult = E_UNEXPECTED; + result_ = operation->GetActivateResult(&activationResult, &activated); + if (SUCCEEDED(result_)) result_ = activationResult; + if (SUCCEEDED(result_)) result_ = activated.As(&client_); + SetEvent(completed_); + return S_OK; + } + + HRESULT Result() const { return result_; } + ComPtr Client() const { return client_; } + + private: + HANDLE completed_ = nullptr; + HRESULT result_ = E_UNEXPECTED; + ComPtr client_; +}; + +struct WindowTarget { + DWORD pid; + UINT64 creationTime; + std::string title; +}; + +HRESULT ReadProcessCreationTime(HANDLE process, UINT64* creationTime) { + FILETIME created{}, exited{}, kernel{}, user{}; + if (!GetProcessTimes(process, &created, &exited, &kernel, &user)) { + return HRESULT_FROM_WIN32(GetLastError()); + } + ULARGE_INTEGER value{}; + value.LowPart = created.dwLowDateTime; + value.HighPart = created.dwHighDateTime; + if (value.QuadPart == 0) return HRESULT_FROM_WIN32(ERROR_INVALID_DATA); + *creationTime = value.QuadPart; + return S_OK; +} + +std::string Utf8(const std::wstring& value) { + if (value.empty()) return {}; + const int size = WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, value.data(), + static_cast(value.size()), nullptr, 0, nullptr, nullptr); + if (size <= 0) return {}; + std::string result(static_cast(size), '\0'); + if (WideCharToMultiByte(CP_UTF8, WC_ERR_INVALID_CHARS, value.data(), + static_cast(value.size()), result.data(), size, nullptr, nullptr) != size) { + return {}; + } + return result; +} + +std::string JsonString(const std::string& value) { + static constexpr char hex[] = "0123456789abcdef"; + std::string result = "\""; + for (const unsigned char character : value) { + switch (character) { + case '\\': result += "\\\\"; break; + case '"': result += "\\\""; break; + case '\b': result += "\\b"; break; + case '\f': result += "\\f"; break; + case '\n': result += "\\n"; break; + case '\r': result += "\\r"; break; + case '\t': result += "\\t"; break; + default: + if (character < 0x20) { + result += "\\u00"; + result += hex[character >> 4]; + result += hex[character & 0xf]; + } else { + result.push_back(static_cast(character)); + } + } + } + result += '"'; + return result; +} + +BOOL CALLBACK CollectWindow(HWND window, LPARAM parameter) { + auto* targets = reinterpret_cast*>(parameter); + if (targets->size() >= kMaxWindows) return FALSE; + if (!IsWindowVisible(window) || GetWindow(window, GW_OWNER) != nullptr) return TRUE; + const int length = GetWindowTextLengthW(window); + if (length <= 0 || length > 512) return TRUE; + DWORD pid = 0; + GetWindowThreadProcessId(window, &pid); + if (pid == 0 || pid == GetCurrentProcessId()) return TRUE; + std::wstring title(static_cast(length) + 1, L'\0'); + const int copied = GetWindowTextW(window, title.data(), length + 1); + if (copied <= 0) return TRUE; + title.resize(static_cast(copied)); + std::string utf8 = Utf8(title); + if (utf8.empty()) return TRUE; + HANDLE process = OpenProcess(PROCESS_QUERY_LIMITED_INFORMATION, FALSE, pid); + if (process == nullptr) return TRUE; + UINT64 creationTime = 0; + const HRESULT result = ReadProcessCreationTime(process, &creationTime); + CloseHandle(process); + if (SUCCEEDED(result)) targets->push_back({pid, creationTime, std::move(utf8)}); + return TRUE; +} + +int ListWindows() { + std::vector targets; + if (!EnumWindows(CollectWindow, reinterpret_cast(&targets))) return 2; + std::cout << '['; + for (size_t index = 0; index < targets.size(); ++index) { + if (index != 0) std::cout << ','; + std::cout << "{\"pid\":" << targets[index].pid << ",\"creationTime\":" + << targets[index].creationTime << ",\"title\":" + << JsonString(targets[index].title) << '}'; + } + std::cout << ']'; + return std::cout.good() ? 0 : 2; +} + +void PutUint32BE(BYTE* output, UINT32 value) { + for (int index = 3; index >= 0; --index) { + output[index] = static_cast(value & 0xff); + value >>= 8; + } +} + +void PutUint64BE(BYTE* output, UINT64 value) { + for (int index = 7; index >= 0; --index) { + output[index] = static_cast(value & 0xff); + value >>= 8; + } +} + +HRESULT WriteAll(HANDLE output, const BYTE* data, DWORD size) { + while (size > 0) { + DWORD written = 0; + if (!WriteFile(output, data, size, &written, nullptr) || written == 0) { + return HRESULT_FROM_WIN32(GetLastError()); + } + data += written; + size -= written; + } + return S_OK; +} + +HRESULT WritePCM(HANDLE output, UINT64 timestamp100ns, const BYTE* data) { + std::array header{}; + header[0] = 'S'; header[1] = 'P'; header[2] = 'C'; header[3] = 'M'; + header[4] = 1; + PutUint64BE(header.data() + 8, timestamp100ns); + PutUint64BE(header.data() + 16, kChunkDuration100ns); + PutUint32BE(header.data() + 24, kBytesPerChunk); + HRESULT result = WriteAll(output, header.data(), static_cast(header.size())); + return SUCCEEDED(result) ? WriteAll(output, data, kBytesPerChunk) : result; +} + +HRESULT ActivateProcessLoopback(DWORD pid, HANDLE completed, ComPtr* client) { + AUDIOCLIENT_ACTIVATION_PARAMS parameters{}; + parameters.ActivationType = AUDIOCLIENT_ACTIVATION_TYPE_PROCESS_LOOPBACK; + parameters.ProcessLoopbackParams.TargetProcessId = pid; + parameters.ProcessLoopbackParams.ProcessLoopbackMode = + PROCESS_LOOPBACK_MODE_INCLUDE_TARGET_PROCESS_TREE; + PROPVARIANT variant{}; + variant.vt = VT_BLOB; + variant.blob.cbSize = sizeof(parameters); + variant.blob.pBlobData = reinterpret_cast(¶meters); + + auto handler = Make(completed); + if (!handler) return E_OUTOFMEMORY; + ComPtr operation; + HRESULT result = ActivateAudioInterfaceAsync(VIRTUAL_AUDIO_DEVICE_PROCESS_LOOPBACK, + __uuidof(IAudioClient), &variant, + handler.Get(), &operation); + if (FAILED(result)) return result; + if (WaitForSingleObject(completed, 10'000) != WAIT_OBJECT_0) return HRESULT_FROM_WIN32(WAIT_TIMEOUT); + result = handler->Result(); + if (SUCCEEDED(result)) *client = handler->Client(); + return result; +} + +HRESULT Capture(DWORD pid, UINT64 expectedCreationTime) { + HANDLE process = OpenProcess(SYNCHRONIZE | PROCESS_QUERY_LIMITED_INFORMATION, FALSE, pid); + if (process == nullptr) return HRESULT_FROM_WIN32(GetLastError()); + HANDLE completed = CreateEventW(nullptr, FALSE, FALSE, nullptr); + HANDLE sampleReady = CreateEventW(nullptr, FALSE, FALSE, nullptr); + if (completed == nullptr || sampleReady == nullptr) { + const HRESULT result = HRESULT_FROM_WIN32(GetLastError()); + if (completed != nullptr) CloseHandle(completed); + if (sampleReady != nullptr) CloseHandle(sampleReady); + CloseHandle(process); + return result; + } + UINT64 creationTime = 0; + HRESULT result = ReadProcessCreationTime(process, &creationTime); + if (SUCCEEDED(result) && creationTime != expectedCreationTime) { + result = HRESULT_FROM_WIN32(ERROR_INVALID_DATA); + } + ComPtr client; + if (SUCCEEDED(result)) result = ActivateProcessLoopback(pid, completed, &client); + ComPtr capture; + WAVEFORMATEX format{}; + format.wFormatTag = WAVE_FORMAT_PCM; + format.nChannels = kChannels; + format.nSamplesPerSec = kSampleRate; + format.wBitsPerSample = kBytesPerSample * 8; + format.nBlockAlign = kChannels * kBytesPerSample; + format.nAvgBytesPerSec = kSampleRate * format.nBlockAlign; + if (SUCCEEDED(result)) { + result = client->Initialize(AUDCLNT_SHAREMODE_SHARED, + AUDCLNT_STREAMFLAGS_LOOPBACK | AUDCLNT_STREAMFLAGS_EVENTCALLBACK | + AUDCLNT_STREAMFLAGS_AUTOCONVERTPCM | AUDCLNT_STREAMFLAGS_SRC_DEFAULT_QUALITY, + 0, 0, &format, nullptr); + } + if (SUCCEEDED(result)) result = client->GetService(IID_PPV_ARGS(&capture)); + if (SUCCEEDED(result)) result = client->SetEventHandle(sampleReady); + if (SUCCEEDED(result)) result = client->Start(); + + std::vector pending; + size_t consumed = 0; + UINT64 nextTimestamp = 0; + HANDLE output = GetStdHandle(STD_OUTPUT_HANDLE); + const HANDLE waits[] = {process, sampleReady}; + while (SUCCEEDED(result)) { + const DWORD wait = WaitForMultipleObjects(2, waits, FALSE, 1'000); + if (wait == WAIT_OBJECT_0) { + result = HRESULT_FROM_WIN32(ERROR_PROCESS_ABORTED); + break; + } + if (wait == WAIT_TIMEOUT) continue; + if (wait != WAIT_OBJECT_0 + 1) { + result = HRESULT_FROM_WIN32(GetLastError()); + break; + } + UINT32 frames = 0; + while (SUCCEEDED(result = capture->GetNextPacketSize(&frames)) && frames > 0) { + BYTE* data = nullptr; + DWORD flags = 0; + UINT64 devicePosition = 0; + UINT64 qpcPosition = 0; + result = capture->GetBuffer(&data, &frames, &flags, &devicePosition, &qpcPosition); + if (FAILED(result)) break; + const size_t bytes = static_cast(frames) * format.nBlockAlign; + if ((flags & AUDCLNT_BUFFERFLAGS_TIMESTAMP_ERROR) != 0) { + capture->ReleaseBuffer(frames); + continue; + } + if ((flags & AUDCLNT_BUFFERFLAGS_DATA_DISCONTINUITY) != 0) { + pending.clear(); + consumed = 0; + } + if (pending.size() == consumed) { + pending.clear(); + consumed = 0; + nextTimestamp = qpcPosition; + } + if ((flags & AUDCLNT_BUFFERFLAGS_SILENT) != 0) { + pending.insert(pending.end(), bytes, 0); + } else { + pending.insert(pending.end(), data, data + bytes); + } + capture->ReleaseBuffer(frames); + while (pending.size() - consumed >= kBytesPerChunk) { + result = WritePCM(output, nextTimestamp, pending.data() + consumed); + if (FAILED(result)) break; + consumed += kBytesPerChunk; + nextTimestamp += kChunkDuration100ns; + } + if (FAILED(result)) break; + if (consumed > kBytesPerChunk * 4) { + pending.erase(pending.begin(), pending.begin() + static_cast(consumed)); + consumed = 0; + } + } + } + if (client) client->Stop(); + CloseHandle(sampleReady); + CloseHandle(completed); + CloseHandle(process); + return result; +} +} // namespace + +int wmain(int argc, wchar_t** argv) { + if (argc == 2 && std::wstring(argv[1]) == L"--list") return ListWindows(); + if (argc != 4 || std::wstring(argv[1]) != L"--capture") return 2; + wchar_t* pidEnd = nullptr; + const unsigned long parsedPID = wcstoul(argv[2], &pidEnd, 10); + wchar_t* creationEnd = nullptr; + const unsigned long long parsedCreationTime = wcstoull(argv[3], &creationEnd, 10); + if (parsedPID == 0 || pidEnd == argv[2] || *pidEnd != L'\0' || + parsedCreationTime == 0 || creationEnd == argv[3] || *creationEnd != L'\0') return 2; + if (_setmode(_fileno(stdout), _O_BINARY) == -1) return 2; + HRESULT result = CoInitializeEx(nullptr, COINIT_MULTITHREADED); + if (FAILED(result)) return 2; + result = Capture(static_cast(parsedPID), static_cast(parsedCreationTime)); + CoUninitialize(); + if (FAILED(result)) std::cerr << "process_audio_capture_failed hresult=0x" << std::hex << result << '\n'; + return SUCCEEDED(result) ? 0 : 2; +} diff --git a/native/sender/go.mod b/native/sender/go.mod index f8bbc89..dc136cf 100644 --- a/native/sender/go.mod +++ b/native/sender/go.mod @@ -7,6 +7,7 @@ require ( github.com/pion/rtcp v1.2.17 github.com/pion/rtp v1.10.5 github.com/pion/webrtc/v4 v4.2.18 + golang.org/x/sys v0.41.0 ) require ( @@ -27,6 +28,5 @@ require ( github.com/wlynxg/anet v0.0.5 // indirect golang.org/x/crypto v0.48.0 // indirect golang.org/x/net v0.50.0 // indirect - golang.org/x/sys v0.41.0 // indirect golang.org/x/time v0.14.0 // indirect ) diff --git a/native/sender/internal/app/server.go b/native/sender/internal/app/server.go index a9b0dfa..485a276 100644 --- a/native/sender/internal/app/server.go +++ b/native/sender/internal/app/server.go @@ -16,6 +16,7 @@ import ( "github.com/TNTcraftHIM/Screener/native/sender/internal/media" "github.com/TNTcraftHIM/Screener/native/sender/internal/remote" + "github.com/TNTcraftHIM/Screener/native/sender/internal/windowaudio" "github.com/coder/websocket" "github.com/coder/websocket/wsjson" ) @@ -59,12 +60,16 @@ type App struct { configTimeout time.Duration activeViewers int waitingViewers int + audioProvider windowaudio.Provider + audioTargets map[string]windowaudio.Target + audioTarget *windowaudio.Target } type startRequest struct { - ServerURL string `json:"serverUrl"` - Password string `json:"password"` - Codec string `json:"codec,omitempty"` + ServerURL string `json:"serverUrl"` + Password string `json:"password"` + Codec string `json:"codec,omitempty"` + AudioTargetID string `json:"audioTargetId,omitempty"` } type encoderConfig struct { @@ -75,6 +80,12 @@ type encoderConfig struct { FPS int `json:"fps"` Bitrate int `json:"bitrate"` EncoderInstances int `json:"encoderInstances"` + Audio bool `json:"audio"` +} + +type localAudioTarget struct { + ID string `json:"id"` + Title string `json:"title"` } func New() (*App, error) { @@ -88,6 +99,8 @@ func New() (*App, error) { attachTimeout: mediaAttachTimeout, configTimeout: mediaConfigTimeout, codec: media.CodecVP8, + audioProvider: windowaudio.NewProvider(), + audioTargets: make(map[string]windowaudio.Target), }, nil } @@ -144,6 +157,8 @@ func (app *App) ServeHTTP(response http.ResponseWriter, request *http.Request) { app.handleStart(response, request) case "/api/stop": app.handleStop(response, request) + case "/api/audio-targets": + app.handleAudioTargets(response, request) case "/media": app.handleMedia(response, request) default: @@ -188,6 +203,16 @@ func (app *App) handleStart(response http.ResponseWriter, request *http.Request) return } app.mu.Lock() + var audioTarget *windowaudio.Target + if input.AudioTargetID != "" { + selected, ok := app.audioTargets[input.AudioTargetID] + if !ok { + app.mu.Unlock() + writeJSON(response, http.StatusBadRequest, map[string]string{"error": "selected audio target is no longer available"}) + return + } + audioTarget = &selected + } if app.starting || app.session != nil { app.mu.Unlock() writeJSON(response, http.StatusConflict, map[string]string{"error": "A sharing session is already active"}) @@ -205,6 +230,7 @@ func (app *App) handleStart(response http.ResponseWriter, request *http.Request) BaseURL: baseURL, HostAdmissionPassword: strings.TrimSpace(input.Password), Codec: codec, + EnableAudio: audioTarget != nil, OnEvent: func(event remote.Event) { app.emit(generation, event) }, }) app.mu.Lock() @@ -219,6 +245,7 @@ func (app *App) handleStart(response http.ResponseWriter, request *http.Request) app.sessionCancel = startCancel app.room = room app.codec = codec + app.audioTarget = audioTarget app.attachTimer = time.AfterFunc(app.attachTimeout, func() { app.mu.Lock() expired := app.generation == generation && app.session == session && app.mediaConn == nil @@ -254,6 +281,35 @@ func (app *App) handleStop(response http.ResponseWriter, request *http.Request) response.WriteHeader(http.StatusNoContent) } +func (app *App) handleAudioTargets(response http.ResponseWriter, request *http.Request) { + if !app.authorizeAPI(response, request, http.MethodPost) { + return + } + targets, err := app.audioProvider.List(request.Context()) + if err != nil { + writeJSON(response, http.StatusOK, map[string]any{ + "supported": false, "targets": []localAudioTarget{}, + "reason": "Windows 11 process audio helper is unavailable", + }) + return + } + localTargets := make([]localAudioTarget, 0, len(targets)) + resolved := make(map[string]windowaudio.Target, len(targets)) + for _, target := range targets { + id, tokenErr := newOpaqueToken() + if tokenErr != nil { + writeJSON(response, http.StatusInternalServerError, map[string]string{"error": "create local audio target identity failed"}) + return + } + resolved[id] = target + localTargets = append(localTargets, localAudioTarget{ID: id, Title: target.Title}) + } + app.mu.Lock() + app.audioTargets = resolved + app.mu.Unlock() + writeJSON(response, http.StatusOK, map[string]any{"supported": true, "targets": localTargets}) +} + func (app *App) handleMedia(response http.ResponseWriter, request *http.Request) { if !exactRequest(request, app.expectedHost, app.origin) || request.Method != http.MethodGet { http.Error(response, "forbidden", http.StatusForbidden) @@ -285,7 +341,7 @@ func (app *App) handleMedia(response http.ResponseWriter, request *http.Request) app.releaseMediaClaim(generation, claimID, nil) return } - connection.SetReadLimit(media.MaxFrameBytes + media.FrameHeaderBytes) + connection.SetReadLimit(media.MaxMediaPayload + media.EnvelopeHeaderBytes) app.mu.Lock() if app.generation != generation || app.session != session || app.mediaClaimID != claimID { app.mu.Unlock() @@ -332,7 +388,8 @@ func (app *App) handleMedia(response http.ResponseWriter, request *http.Request) return } var config encoderConfig - if err = decodeLocalPayload(payload, &config); err != nil || !validEncoderConfigForCodec(config, session.Codec()) { + if err = decodeLocalPayload(payload, &config); err != nil || + !validEncoderConfigForCodec(config, session.Codec()) || config.Audio != session.AudioEnabled() { session.Fail(errors.New("local encoder configuration is invalid")) _ = connection.Close(websocket.StatusPolicyViolation, "invalid encoder config") return @@ -341,6 +398,39 @@ func (app *App) handleMedia(response http.ResponseWriter, request *http.Request) session.Fail(fmt.Errorf("acknowledge local encoder configuration: %w", err)) return } + app.mu.Lock() + target := app.audioTarget + app.mu.Unlock() + captureContext, cancelCapture := context.WithCancel(app.ctx) + audioDone := make(chan struct{}) + if target != nil { + _ = app.writeLocal(connection, map[string]string{"kind": "audio-state", "state": "starting"}) + go func(selected windowaudio.Target) { + defer close(audioDone) + captureErr := app.audioProvider.Capture(captureContext, selected, func(chunk windowaudio.PCMChunk) error { + envelope, encodeErr := media.EncodePacket(media.Packet{ + Kind: media.KindPCM, Timestamp100ns: chunk.Timestamp100ns, + Duration100ns: chunk.Duration100ns, Data: chunk.Data, + }) + if encodeErr != nil { + return encodeErr + } + return app.writeLocalBinary(connection, envelope) + }) + if captureErr != nil && captureContext.Err() == nil { + _ = app.writeLocal(connection, map[string]string{ + "kind": "audio-state", "state": "unavailable", + "message": "Target audio stopped; system audio was not substituted", + }) + } + }(*target) + } else { + close(audioDone) + } + defer func() { + cancelCapture() + <-audioDone + }() for { messageType, payload, err = connection.Read(app.ctx) if err != nil { @@ -354,13 +444,18 @@ func (app *App) handleMedia(response http.ResponseWriter, request *http.Request) _ = connection.Close(websocket.StatusUnsupportedData, "binary frames required") return } - frame, decodeErr := media.DecodeFrame(payload) + packet, decodeErr := media.DecodePacket(payload) if decodeErr != nil { - session.Fail(fmt.Errorf("decode local encoded frame: %w", decodeErr)) - _ = connection.Close(websocket.StatusPolicyViolation, "invalid encoded frame") + session.Fail(fmt.Errorf("decode local encoded media: %w", decodeErr)) + _ = connection.Close(websocket.StatusPolicyViolation, "invalid encoded media") return } - if err = session.WriteFrame(frame); err != nil { + if packet.Kind != media.KindVideo && packet.Kind != media.KindOpus { + session.Fail(errors.New("local media bridge received an invalid encoded media kind")) + _ = connection.Close(websocket.StatusPolicyViolation, "encoded media required") + return + } + if err = session.WriteMedia(packet); err != nil { return } } @@ -426,6 +521,14 @@ func (app *App) writeLocal(connection *websocket.Conn, value any) error { return wsjson.Write(ctx, connection, value) } +func (app *App) writeLocalBinary(connection *websocket.Conn, payload []byte) error { + app.writeMu.Lock() + defer app.writeMu.Unlock() + ctx, cancel := context.WithTimeout(app.ctx, localWriteTimeout) + defer cancel() + return connection.Write(ctx, websocket.MessageBinary, payload) +} + func (app *App) releaseMediaClaim(generation, claimID uint64, connection *websocket.Conn) { app.mu.Lock() defer app.mu.Unlock() @@ -465,6 +568,7 @@ func (app *App) stopActiveGeneration(expectedGeneration uint64) { } app.room = remote.Room{} app.codec = media.CodecVP8 + app.audioTarget = nil app.activeViewers = 0 app.waitingViewers = 0 app.mu.Unlock() diff --git a/native/sender/internal/app/server_test.go b/native/sender/internal/app/server_test.go index c6926f6..8f4544a 100644 --- a/native/sender/internal/app/server_test.go +++ b/native/sender/internal/app/server_test.go @@ -3,7 +3,6 @@ package app import ( "bytes" "context" - "encoding/binary" "encoding/json" "errors" "net/http" @@ -15,6 +14,7 @@ import ( "github.com/TNTcraftHIM/Screener/native/sender/internal/media" "github.com/TNTcraftHIM/Screener/native/sender/internal/remote" + "github.com/TNTcraftHIM/Screener/native/sender/internal/windowaudio" "github.com/coder/websocket" ) @@ -62,6 +62,49 @@ func TestDecodeLocalPayloadIsStrictAndSingleValued(t *testing.T) { } } +func TestAudioTargetListingKeepsProcessIdentityBehindAnOpaqueLocalID(t *testing.T) { + application, err := New() + if err != nil { + t.Fatal(err) + } + application.audioProvider = fakeAudioProvider{targets: []windowaudio.Target{{ + PID: 424_242, CreationTime: 987_654_321, Title: "Game Window", + }}} + if _, err = application.Start(); err != nil { + t.Fatal(err) + } + defer application.Close() + + request, err := http.NewRequest(http.MethodPost, application.origin+"/api/audio-targets", nil) + if err != nil { + t.Fatal(err) + } + request.Header.Set("Origin", application.origin) + request.Header.Set(processTokenHeader, application.token) + response, err := http.DefaultClient.Do(request) + if err != nil { + t.Fatal(err) + } + defer response.Body.Close() + var result struct { + Supported bool `json:"supported"` + Targets []localAudioTarget `json:"targets"` + } + if json.NewDecoder(response.Body).Decode(&result) != nil || !result.Supported || len(result.Targets) != 1 { + t.Fatalf("audio target response = %+v", result) + } + if result.Targets[0].ID == "" || result.Targets[0].Title != "Game Window" || + strings.Contains(result.Targets[0].ID, "424242") || strings.Contains(result.Targets[0].ID, "987654321") { + t.Fatalf("audio target was not locally anonymized: %+v", result.Targets[0]) + } + application.mu.Lock() + resolved := application.audioTargets[result.Targets[0].ID] + application.mu.Unlock() + if resolved.PID != 424_242 || resolved.CreationTime != 987_654_321 { + t.Fatal("opaque target did not resolve in process memory") + } +} + func TestStaleSessionGenerationCannotMutateOrStopTheCurrentSession(t *testing.T) { application := &App{generation: 7, activeViewers: 1, waitingViewers: 2} application.emit(6, remote.Event{Kind: "viewer-count", ActiveViewers: 9, WaitingViewers: 9}) @@ -193,19 +236,26 @@ func TestConfiguredMediaFramesWithCaptureJitterReachFanout(t *testing.T) { application, events, terminal := startObservedTestApplication(t) connection := openConfiguredMedia(t, application) - payload := make([]byte, media.FrameHeaderBytes+4) - payload[0] = 1 - binary.BigEndian.PutUint64(payload[1:9], 1_000_000) - binary.BigEndian.PutUint64(payload[9:17], 33_333) - copy(payload[media.FrameHeaderBytes:], []byte{0x10, 0x00, 0x00, 0x00}) + payload, err := media.EncodePacket(media.Packet{ + Kind: media.KindVideo, KeyFrame: true, Timestamp100ns: 10_000_000, + Duration100ns: 333_330, Data: []byte{0x10, 0x00, 0x00, 0x00}, + }) + if err != nil { + t.Fatal(err) + } writeContext, cancelWrite := context.WithTimeout(context.Background(), time.Second) - err := connection.Write(writeContext, websocket.MessageBinary, payload) + err = connection.Write(writeContext, websocket.MessageBinary, payload) cancelWrite() if err != nil { t.Fatalf("write encoded frame: %v", err) } - payload[0] = 2 - binary.BigEndian.PutUint64(payload[1:9], 1_033_000) + payload, err = media.EncodePacket(media.Packet{ + Kind: media.KindVideo, Timestamp100ns: 10_330_000, + Duration100ns: 333_330, Data: []byte{0x10, 0x00, 0x00, 0x00}, + }) + if err != nil { + t.Fatal(err) + } writeContext, cancelWrite = context.WithTimeout(context.Background(), time.Second) err = connection.Write(writeContext, websocket.MessageBinary, payload) cancelWrite() @@ -638,3 +688,13 @@ func appRoomResponse(origin string) map[string]any { "viewerPolicy": "private-link", "viewerGrantExpiresAt": expiresAt.Format(time.RFC3339), "expiresAt": expiresAt.Format(time.RFC3339), } } + +type fakeAudioProvider struct{ targets []windowaudio.Target } + +func (provider fakeAudioProvider) List(context.Context) ([]windowaudio.Target, error) { + return provider.targets, nil +} + +func (fakeAudioProvider) Capture(context.Context, windowaudio.Target, func(windowaudio.PCMChunk) error) error { + return nil +} diff --git a/native/sender/internal/app/ui/app.js b/native/sender/internal/app/ui/app.js index 3215f5f..a5a8ffc 100644 --- a/native/sender/internal/app/ui/app.js +++ b/native/sender/internal/app/ui/app.js @@ -12,6 +12,10 @@ const FRAME_DURATION = Math.round(1_000_000 / FPS); const MAX_ENCODER_QUEUE = 2; const MAX_SOCKET_BUFFER = 512 * 1024; const MAX_FRAME_BYTES = 1 << 20; +const ENVELOPE_HEADER_BYTES = 20; +const MEDIA_KIND_VIDEO = 1; +const MEDIA_KIND_PCM = 2; +const MEDIA_KIND_OPUS = 3; const fragment = new URLSearchParams(location.hash.slice(1)); const processToken = fragment.get("token") || ""; @@ -21,6 +25,8 @@ const elements = { serverURL: document.querySelector("#server-url"), password: document.querySelector("#password"), codec: document.querySelector("#codec"), + audioTarget: document.querySelector("#audio-target"), + refreshAudio: document.querySelector("#refresh-audio"), start: document.querySelector("#start"), stop: document.querySelector("#stop"), preview: document.querySelector("#preview"), @@ -36,11 +42,14 @@ const elements = { bridgeDiagnostic: document.querySelector("#bridge-diagnostic"), helperDiagnostic: document.querySelector("#helper-diagnostic"), edgeDiagnostic: document.querySelector("#edge-diagnostic"), + audioStatus: document.querySelector("#audio-status"), }; let stream = null; let processorReader = null; let encoder = null; +let audioEncoder = null; +let silentAudioChunks = 0; let mediaSocket = null; let sharing = false; let stopping = false; @@ -71,6 +80,8 @@ elements.copy.addEventListener("click", async () => { showError("无法复制链接，请直接打开邀请链接"); } }); +elements.refreshAudio.addEventListener("click", () => void loadAudioTargets()); +if (processToken) void loadAudioTargets(); async function startSharing() { if (sharing || stopping || startInFlight) return; @@ -83,6 +94,13 @@ async function startSharing() { elements.start.disabled = true; const generation = ++startGeneration; const codec = selectedCodec(); + const audioTargetId = elements.audioTarget.value; + if (audioTargetId && (!("AudioEncoder" in window) || !("AudioData" in window))) { + showError("当前浏览器缺少 WebCodecs Opus 编码支持，请关闭目标进程音频或使用当前版 Chrome / Edge"); + startInFlight = false; + elements.start.disabled = false; + return; + } setStatus("选择屏幕"); try { const capturedStream = await navigator.mediaDevices.getDisplayMedia({ @@ -111,6 +129,7 @@ async function startSharing() { serverUrl: elements.serverURL.value, password: elements.password.value, codec, + audioTargetId, }); assertCurrentStart(generation); if (pendingStartGeneration === generation) pendingStartGeneration = 0; @@ -120,7 +139,8 @@ async function startSharing() { elements.inviteLink.textContent = room.inviteUrl; elements.invite.hidden = false; - await connectMediaSocket(generation, codec); + if (audioTargetId) await startAudioEncoder(generation); + await connectMediaSocket(generation, codec, Boolean(audioTargetId)); assertCurrentStart(generation); const activeEncoder = await startEncoder(generation, codec); assertCurrentStart(generation); @@ -139,7 +159,7 @@ async function startSharing() { } } -async function connectMediaSocket(generation, codec) { +async function connectMediaSocket(generation, codec, audio) { const url = new URL("/media", location.href); url.protocol = "ws:"; const socket = new WebSocket(url, [`screener.token.${processToken}`]); @@ -166,6 +186,7 @@ async function connectMediaSocket(generation, codec) { fps: FPS, bitrate: BITRATE, encoderInstances: 1, + audio, })); await accepted; assertCurrentStart(generation, socket); @@ -275,12 +296,13 @@ function sendEncodedChunk(generation, activeEncoder, chunk) { } const data = new Uint8Array(chunk.byteLength); chunk.copyTo(data); - const message = new ArrayBuffer(17 + data.byteLength); - const view = new DataView(message); - view.setUint8(0, chunk.type === "key" ? 1 : 2); - view.setBigUint64(1, BigInt(Math.max(0, chunk.timestamp)), false); - view.setBigUint64(9, BigInt(chunk.duration || FRAME_DURATION), false); - new Uint8Array(message, 17).set(data); + const message = mediaEnvelope( + MEDIA_KIND_VIDEO, + chunk.type === "key" ? 1 : 0, + BigInt(Math.max(0, chunk.timestamp)) * 10n, + BigInt(chunk.duration || FRAME_DURATION) * 10n, + data, + ); mediaSocket.send(message); metrics.encoderOutputs += 1; metrics.encodedBytes += data.byteLength; @@ -293,6 +315,10 @@ function sendEncodedChunk(generation, activeEncoder, chunk) { function handleLocalEvent(generation, socket, event) { if (generation !== startGeneration || socket !== mediaSocket) return; + if (event.data instanceof ArrayBuffer) { + handlePCM(generation, event.data); + return; + } if (typeof event.data !== "string") return; const message = parseLocalMessage(event.data); if (!message) return; @@ -305,6 +331,11 @@ function handleLocalEvent(generation, socket, event) { updateDiagnostics(); } if (message.kind === "warning" && message.message) showError(message.message); + if (message.kind === "audio-state") { + elements.audioStatus.textContent = message.state === "unavailable" + ? "不可用（未回退系统声音）" + : "正在连接目标进程"; + } if (message.kind === "fatal") { void failSharing(generation, message.message || "分享会话已失败并停止"); } @@ -347,6 +378,12 @@ async function stopSharing(expectedGeneration = startGeneration) { rememberError(error); } encoder = null; + try { + if (audioEncoder && audioEncoder.state !== "closed") audioEncoder.close(); + } catch (error) { + rememberError(error); + } + audioEncoder = null; const socket = mediaSocket; mediaSocket = null; try { @@ -365,6 +402,7 @@ async function stopSharing(expectedGeneration = startGeneration) { } finally { frameNumber = 0; forceKeyFrame = true; + silentAudioChunks = 0; captureSettings = null; nativeDiagnostics = null; metrics = emptyMetrics(); @@ -379,6 +417,7 @@ async function stopSharing(expectedGeneration = startGeneration) { elements.start.disabled = startInFlight; showViewerCounts(0, 0); setStatus("未开始"); + elements.audioStatus.textContent = "关闭"; stopping = false; } } @@ -446,6 +485,100 @@ async function localRequest(path, body) { return result; } +async function loadAudioTargets() { + elements.refreshAudio.disabled = true; + const previous = elements.audioTarget.value; + try { + const result = await localRequest("/api/audio-targets"); + elements.audioTarget.replaceChildren(new Option("关闭", "")); + for (const target of Array.isArray(result.targets) ? result.targets : []) { + if (target?.id && target?.title) elements.audioTarget.add(new Option(target.title, target.id)); + } + if ([...elements.audioTarget.options].some((option) => option.value === previous)) { + elements.audioTarget.value = previous; + } + elements.audioTarget.disabled = !result.supported; + elements.audioStatus.textContent = result.supported ? "关闭" : "Windows 11 helper 不可用"; + } catch (error) { + elements.audioTarget.disabled = true; + elements.audioStatus.textContent = readableError(error, "窗口列表不可用"); + } finally { + elements.refreshAudio.disabled = false; + } +} + +async function startAudioEncoder(generation) { + const config = { codec: "opus", sampleRate: 48_000, numberOfChannels: 2, bitrate: 128_000 }; + const support = await AudioEncoder.isConfigSupported(config); + assertCurrentStart(generation); + if (!support.supported) throw new Error("当前浏览器不支持 WebCodecs Opus 编码"); + let active; + active = new AudioEncoder({ + output: (chunk) => sendEncodedAudio(generation, active, chunk), + error: (error) => void failSharing(generation, `音频编码器失败：${error.message}`), + }); + audioEncoder = active; + active.configure(support.config); + elements.audioStatus.textContent = "等待目标进程音频"; +} + +function handlePCM(generation, message) { + if (generation !== startGeneration || !audioEncoder || audioEncoder.state !== "configured") return; + if (audioEncoder.encodeQueueSize >= 4) return; + try { + const view = new DataView(message); + if (message.byteLength <= ENVELOPE_HEADER_BYTES || view.getUint8(0) !== 1 || + view.getUint8(1) !== MEDIA_KIND_PCM || view.getUint8(2) !== 0 || view.getUint8(3) !== 0) { + throw new Error("本地 PCM envelope 无效"); + } + const data = new Uint8Array(message, ENVELOPE_HEADER_BYTES); + if (data.byteLength % 4 !== 0) throw new Error("本地 PCM 帧长度无效"); + let audible = false; + for (const byte of data) if (byte !== 0) { audible = true; break; } + silentAudioChunks = audible ? 0 : silentAudioChunks + 1; + elements.audioStatus.textContent = silentAudioChunks >= 100 + ? "目标进程当前无声（未回退系统声音）" + : "目标进程树音频"; + const audio = new AudioData({ + format: "s16", sampleRate: 48_000, numberOfChannels: 2, + numberOfFrames: data.byteLength / 4, + timestamp: Number(view.getBigUint64(4, false) / 10n), data, + }); + audioEncoder.encode(audio); + audio.close(); + } catch (error) { + void failSharing(generation, readableError(error, "处理目标进程音频失败")); + } +} + +function sendEncodedAudio(generation, activeEncoder, chunk) { + if (generation !== startGeneration || stopping || audioEncoder !== activeEncoder || + !mediaSocket || mediaSocket.readyState !== WebSocket.OPEN) return; + if (chunk.byteLength < 1 || chunk.byteLength > 64 * 1024 || + mediaSocket.bufferedAmount > MAX_SOCKET_BUFFER) { + void failSharing(generation, "本地音频桥接超过安全上限"); + return; + } + const data = new Uint8Array(chunk.byteLength); + chunk.copyTo(data); + mediaSocket.send(mediaEnvelope( + MEDIA_KIND_OPUS, 0, BigInt(Math.max(0, chunk.timestamp)) * 10n, + BigInt(chunk.duration || 20_000) * 10n, data, + )); +} + +function mediaEnvelope(kind, flags, timestamp100ns, duration100ns, data) { + const message = new ArrayBuffer(ENVELOPE_HEADER_BYTES + data.byteLength); + const view = new DataView(message); + view.setUint8(0, 1); + view.setUint8(1, kind); + view.setUint8(2, flags); + view.setBigUint64(4, timestamp100ns, false); + view.setBigUint64(12, duration100ns, false); + new Uint8Array(message, ENVELOPE_HEADER_BYTES).set(data); + return message; +} + function showViewerCounts(activeValue, waitingValue) { const active = boundedCount(activeValue); const waiting = boundedCount(waitingValue); diff --git a/native/sender/internal/app/ui/index.html b/native/sender/internal/app/ui/index.html index 199e897..2a75ffa 100644 --- a/native/sender/internal/app/ui/index.html +++ b/native/sender/internal/app/ui/index.html @@ -45,8 +45,15 @@ + +

请在浏览器画面选择器中选择同一目标。音频只包含所选进程树，失败时不会改用系统混音。

-

当前 canary 仅发送画面，不含音频。

技术信息 @@ -57,7 +64,7 @@
本地桥接
尚无数据
Helper
尚无数据
出边
无活动媒体边
-
音频
当前 canary 未启用，电影与视频音质仍待验收
+
音频
关闭
diff --git a/native/sender/internal/app/ui/styles.css b/native/sender/internal/app/ui/styles.css index 062d57a..7d6da52 100644 --- a/native/sender/internal/app/ui/styles.css +++ b/native/sender/internal/app/ui/styles.css @@ -16,8 +16,8 @@ * { box-sizing: border-box; } html, body { min-width: 320px; min-height: 100%; margin: 0; } body { min-height: 100vh; background: #101310; } -button, input { font: inherit; letter-spacing: 0; } -button:focus-visible, input:focus-visible, a:focus-visible, summary:focus-visible { outline: 2px solid var(--green); outline-offset: 2px; } +button, input, select { font: inherit; letter-spacing: 0; } +button:focus-visible, input:focus-visible, select:focus-visible, a:focus-visible, summary:focus-visible { outline: 2px solid var(--green); outline-offset: 2px; } .app-header { display: flex; @@ -44,7 +44,9 @@ video { display: block; width: 100%; height: 100%; object-fit: contain; } .controls { display: grid; align-content: start; gap: 18px; padding: 32px 24px; border-left: 1px solid var(--border); background: var(--surface); } label { display: grid; gap: 8px; color: var(--muted); font-size: 13px; } -input { width: 100%; min-height: 44px; padding: 10px 11px; border: 1px solid var(--border); border-radius: 6px; color: #f1f3f1; background: #101310; } +input, select { width: 100%; min-width: 0; min-height: 44px; padding: 10px 11px; border: 1px solid var(--border); border-radius: 6px; color: #f1f3f1; background: #101310; } +.select-row { display: grid; grid-template-columns: minmax(0, 1fr) 44px; gap: 8px; } +.audio-note { margin: -8px 0 0; color: var(--muted); font-size: 12px; line-height: 1.45; } .button, .icon-button { min-height: 44px; border: 1px solid transparent; border-radius: 6px; color: inherit; cursor: pointer; } .button:disabled { cursor: not-allowed; opacity: .5; } .primary { color: #07150c; font-weight: 700; background: var(--green); } diff --git a/native/sender/internal/media/audio.go b/native/sender/internal/media/audio.go new file mode 100644 index 0000000..513816d --- /dev/null +++ b/native/sender/internal/media/audio.go @@ -0,0 +1,178 @@ +package media + +import ( + "context" + "errors" + "fmt" + "math" + "sync" + + "github.com/pion/rtp" + "github.com/pion/rtp/codecs" + "github.com/pion/webrtc/v4" +) + +const ( + audioClockRate = 48_000 + audioQueueDepth = 16 + OpusSDPFmtpLine = "minptime=10;useinbandfec=1" +) + +type AudioMetrics struct { + PacketsWritten uint64 `json:"packetsWritten"` + SourceRTPPacketsWritten uint64 `json:"sourceRtpPacketsWritten"` + SourceRTPBytesWritten uint64 `json:"sourceRtpBytesWritten"` + LastSourceTimestamp100ns uint64 `json:"lastSourceTimestamp100ns"` +} + +type AudioFanout struct { + track *webrtc.TrackLocalStaticRTP + writer rtpWriter + queue chan Packet + ctx context.Context + cancel context.CancelFunc + done chan struct{} + closeOnce sync.Once + fatalOnce sync.Once + onFatal func(error) + metricsMu sync.Mutex + metrics AudioMetrics +} + +func NewAudioFanout(onFatal func(error)) (*AudioFanout, error) { + capability := webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeOpus, ClockRate: audioClockRate, Channels: 2, SDPFmtpLine: OpusSDPFmtpLine, + } + track, err := webrtc.NewTrackLocalStaticRTP(capability, "audio", "screener") + if err != nil { + return nil, fmt.Errorf("create shared Opus track: %w", err) + } + return newAudioFanout(track, track, onFatal), nil +} + +func newAudioFanout(track *webrtc.TrackLocalStaticRTP, writer rtpWriter, onFatal func(error)) *AudioFanout { + ctx, cancel := context.WithCancel(context.Background()) + fanout := &AudioFanout{ + track: track, writer: writer, queue: make(chan Packet, audioQueueDepth), + ctx: ctx, cancel: cancel, done: make(chan struct{}), onFatal: onFatal, + } + go fanout.writeLoop() + return fanout +} + +func (fanout *AudioFanout) Track() *webrtc.TrackLocalStaticRTP { return fanout.track } + +func (fanout *AudioFanout) Push(packet Packet) error { + if packet.Kind != KindOpus { + return errors.New("shared audio fanout requires Opus") + } + if err := validatePacket(packet); err != nil { + return err + } + select { + case <-fanout.ctx.Done(): + return errors.New("shared audio fanout is closed") + case fanout.queue <- packet: + return nil + default: + return errors.New("shared audio queue exceeded its bound") + } +} + +func (fanout *AudioFanout) Snapshot() AudioMetrics { + fanout.metricsMu.Lock() + defer fanout.metricsMu.Unlock() + return fanout.metrics +} + +func (fanout *AudioFanout) Close() { + fanout.stop() + <-fanout.done +} + +func (fanout *AudioFanout) writeLoop() { + defer close(fanout.done) + timeline := newAudioTimeline() + for { + select { + case <-fanout.ctx.Done(): + return + case packet := <-fanout.queue: + packets, err := timeline.Packetize(packet) + if err != nil { + fanout.fail(err) + return + } + var sourceBytes uint64 + for _, rtpPacket := range packets { + if err = fanout.writer.WriteRTP(rtpPacket); err != nil { + fanout.fail(errors.New("write shared Opus RTP packet failed")) + return + } + sourceBytes += uint64(rtpPacket.MarshalSize()) + } + fanout.metricsMu.Lock() + fanout.metrics.PacketsWritten++ + fanout.metrics.SourceRTPPacketsWritten += uint64(len(packets)) + fanout.metrics.SourceRTPBytesWritten += sourceBytes + fanout.metrics.LastSourceTimestamp100ns = packet.Timestamp100ns + fanout.metricsMu.Unlock() + } + } +} + +func (fanout *AudioFanout) fail(err error) { + fanout.fatalOnce.Do(func() { + fanout.stop() + if fanout.onFatal != nil { + go func() { <-fanout.done; fanout.onFatal(err) }() + } + }) +} + +func (fanout *AudioFanout) stop() { fanout.closeOnce.Do(fanout.cancel) } + +type audioTimeline struct { + packetizer rtp.Packetizer + hasPrevious bool + previousTimestamp uint64 +} + +func newAudioTimeline() *audioTimeline { + return &audioTimeline{packetizer: rtp.NewPacketizer( + 1200, 111, 0, &codecs.OpusPayloader{}, rtp.NewRandomSequencer(), audioClockRate, + )} +} + +func (timeline *audioTimeline) Packetize(packet Packet) ([]*rtp.Packet, error) { + if timeline.hasPrevious { + if packet.Timestamp100ns <= timeline.previousTimestamp { + return nil, errors.New("encoded audio timestamp did not increase") + } + delta, err := audioSamples(packet.Timestamp100ns - timeline.previousTimestamp) + if err != nil { + return nil, err + } + if delta == 0 { + delta = 1 + } + timeline.packetizer.SkipSamples(delta) + } + packets := timeline.packetizer.Packetize(packet.Data, 0) + if len(packets) == 0 { + return nil, errors.New("Opus access unit exceeded the RTP payload bound") + } + timeline.hasPrevious = true + timeline.previousTimestamp = packet.Timestamp100ns + return packets, nil +} + +func audioSamples(value100ns uint64) (uint32, error) { + seconds := value100ns / 10_000_000 + remainder := value100ns % 10_000_000 + samples := seconds*audioClockRate + (remainder*audioClockRate+5_000_000)/10_000_000 + if samples > math.MaxUint32 { + return 0, errors.New("encoded audio timestamp gap is too large") + } + return uint32(samples), nil +} diff --git a/native/sender/internal/media/audio_test.go b/native/sender/internal/media/audio_test.go new file mode 100644 index 0000000..3c63ff6 --- /dev/null +++ b/native/sender/internal/media/audio_test.go @@ -0,0 +1,56 @@ +package media + +import ( + "testing" + "time" + + "github.com/pion/rtp" + "github.com/pion/webrtc/v4" +) + +func TestAudioFanoutUsesOneOpusTimeline(t *testing.T) { + written := make(chan *rtp.Packet, 2) + fanout := newAudioFanout(nil, rtpWriterFunc(func(packet *rtp.Packet) error { + written <- packet.Clone() + return nil + }), nil) + defer fanout.Close() + + for _, timestamp := range []uint64{100_000, 300_000} { + if err := fanout.Push(Packet{ + Kind: KindOpus, Timestamp100ns: timestamp, Duration100ns: 200_000, + Data: []byte{0xf8, 0xff, 0xfe}, + }); err != nil { + t.Fatal(err) + } + } + first := awaitAudioPacket(t, written) + second := awaitAudioPacket(t, written) + if delta := second.Timestamp - first.Timestamp; delta != 960 { + t.Fatalf("Opus RTP timestamp delta = %d, want 960", delta) + } +} + +func TestOpusTrackCapabilityIsBrowserCompatible(t *testing.T) { + fanout, err := NewAudioFanout(nil) + if err != nil { + t.Fatal(err) + } + defer fanout.Close() + codec := fanout.Track().Codec() + if codec.MimeType != webrtc.MimeTypeOpus || codec.ClockRate != 48_000 || codec.Channels != 2 || + codec.SDPFmtpLine != OpusSDPFmtpLine { + t.Fatalf("Opus capability = %+v", codec) + } +} + +func awaitAudioPacket(t *testing.T, packets <-chan *rtp.Packet) *rtp.Packet { + t.Helper() + select { + case packet := <-packets: + return packet + case <-time.After(time.Second): + t.Fatal("timed out waiting for shared Opus RTP") + return nil + } +} diff --git a/native/sender/internal/media/envelope.go b/native/sender/internal/media/envelope.go new file mode 100644 index 0000000..87cb53a --- /dev/null +++ b/native/sender/internal/media/envelope.go @@ -0,0 +1,104 @@ +package media + +import ( + "encoding/binary" + "errors" + "time" +) + +const ( + EnvelopeVersion = 1 + EnvelopeHeaderBytes = 20 + MaxMediaPayload = 1 << 20 + MaxPCMBytes = 48_000 * 2 * 2 / 10 // 100 ms of 48 kHz stereo s16. + + flagKeyFrame = 1 << 0 +) + +type Kind uint8 + +const ( + KindVideo Kind = 1 + KindPCM Kind = 2 + KindOpus Kind = 3 +) + +type Packet struct { + Kind Kind + KeyFrame bool + Timestamp100ns uint64 + Duration100ns uint64 + Data []byte +} + +func EncodePacket(packet Packet) ([]byte, error) { + if err := validatePacket(packet); err != nil { + return nil, err + } + payload := make([]byte, EnvelopeHeaderBytes+len(packet.Data)) + payload[0] = EnvelopeVersion + payload[1] = byte(packet.Kind) + if packet.KeyFrame { + payload[2] = flagKeyFrame + } + binary.BigEndian.PutUint64(payload[4:12], packet.Timestamp100ns) + binary.BigEndian.PutUint64(payload[12:20], packet.Duration100ns) + copy(payload[EnvelopeHeaderBytes:], packet.Data) + return payload, nil +} + +func DecodePacket(payload []byte) (Packet, error) { + if len(payload) <= EnvelopeHeaderBytes || len(payload)-EnvelopeHeaderBytes > MaxMediaPayload { + return Packet{}, errors.New("media envelope is empty, truncated, or too large") + } + if payload[0] != EnvelopeVersion || payload[3] != 0 || payload[2]&^byte(flagKeyFrame) != 0 { + return Packet{}, errors.New("media envelope header is invalid") + } + packet := Packet{ + Kind: Kind(payload[1]), + KeyFrame: payload[2]&flagKeyFrame != 0, + Timestamp100ns: binary.BigEndian.Uint64(payload[4:12]), + Duration100ns: binary.BigEndian.Uint64(payload[12:20]), + Data: append([]byte(nil), payload[EnvelopeHeaderBytes:]...), + } + if err := validatePacket(packet); err != nil { + return Packet{}, err + } + return packet, nil +} + +func (packet Packet) VideoFrame() (Frame, error) { + if packet.Kind != KindVideo { + return Frame{}, errors.New("media envelope is not encoded video") + } + return Frame{ + KeyFrame: packet.KeyFrame, + TimestampMicros: packet.Timestamp100ns / 10, + DurationMicros: packet.Duration100ns / 10, + Data: packet.Data, + }, nil +} + +func validatePacket(packet Packet) error { + if len(packet.Data) == 0 || len(packet.Data) > MaxMediaPayload || packet.Duration100ns == 0 || + packet.Duration100ns > uint64(time.Second/(100*time.Nanosecond)) { + return errors.New("media envelope violates the bounded media contract") + } + switch packet.Kind { + case KindVideo: + if packet.Duration100ns < 10 || packet.Duration100ns%10 != 0 { + return errors.New("video envelope timestamp unit is invalid") + } + case KindPCM: + if packet.KeyFrame || len(packet.Data) > MaxPCMBytes || len(packet.Data)%4 != 0 { + return errors.New("PCM envelope is invalid") + } + case KindOpus: + if packet.KeyFrame || len(packet.Data) > 64<<10 { + return errors.New("Opus envelope is invalid") + } + default: + return errors.New("media envelope kind is invalid") + } + return nil +} diff --git a/native/sender/internal/media/fanout.go b/native/sender/internal/media/fanout.go index 598cbf7..ed61346 100644 --- a/native/sender/internal/media/fanout.go +++ b/native/sender/internal/media/fanout.go @@ -2,12 +2,10 @@ package media import ( "context" - "encoding/binary" "errors" "fmt" "math" "sync" - "time" "github.com/pion/rtp" "github.com/pion/rtp/codecs" @@ -15,10 +13,8 @@ import ( ) const ( - FrameHeaderBytes = 17 - MaxFrameBytes = 1 << 20 - queueCapacity = 8 - videoClockRate = 90_000 + queueCapacity = 8 + videoClockRate = 90_000 ) // Codec selects the encoded representation carried by the native bridge. @@ -135,7 +131,7 @@ func (fanout *Fanout) Track() *webrtc.TrackLocalStaticRTP { } func (fanout *Fanout) Push(frame Frame) error { - if len(frame.Data) == 0 || len(frame.Data) > MaxFrameBytes || frame.DurationMicros == 0 { + if len(frame.Data) == 0 || len(frame.Data) > MaxMediaPayload || frame.DurationMicros == 0 { return errors.New("encoded frame violates the bounded media contract") } result, err := fanout.queue.Push(frame) @@ -283,25 +279,3 @@ func microsToSamples(micros uint64) (uint32, error) { } return uint32(samples), nil } - -func DecodeFrame(payload []byte) (Frame, error) { - if len(payload) <= FrameHeaderBytes || len(payload)-FrameHeaderBytes > MaxFrameBytes { - return Frame{}, errors.New("encoded frame is empty, truncated, or too large") - } - frame := Frame{ - TimestampMicros: binary.BigEndian.Uint64(payload[1:9]), - DurationMicros: binary.BigEndian.Uint64(payload[9:17]), - Data: append([]byte(nil), payload[FrameHeaderBytes:]...), - } - switch payload[0] { - case 1: - frame.KeyFrame = true - case 2: - default: - return Frame{}, errors.New("encoded frame type is invalid") - } - if frame.DurationMicros == 0 || frame.DurationMicros > uint64(time.Second/time.Microsecond) { - return Frame{}, errors.New("encoded frame duration is invalid") - } - return frame, nil -} diff --git a/native/sender/internal/media/fanout_test.go b/native/sender/internal/media/fanout_test.go index e80e05d..9285b95 100644 --- a/native/sender/internal/media/fanout_test.go +++ b/native/sender/internal/media/fanout_test.go @@ -1,7 +1,6 @@ package media import ( - "encoding/binary" "errors" "runtime" "sync" @@ -207,14 +206,19 @@ func TestFanoutCloseDropsBacklogWithoutLateWritesOrFatal(t *testing.T) { } } -func TestDecodeFrame(t *testing.T) { - payload := make([]byte, FrameHeaderBytes+3) - payload[0] = 1 - binary.BigEndian.PutUint64(payload[1:9], 123) - binary.BigEndian.PutUint64(payload[9:17], 33_333) - copy(payload[17:], []byte{1, 2, 3}) - - frame, err := DecodeFrame(payload) +func TestTypedEnvelopePreservesVideoTimeline(t *testing.T) { + payload, err := EncodePacket(Packet{ + Kind: KindVideo, KeyFrame: true, Timestamp100ns: 1_230, + Duration100ns: 333_330, Data: []byte{1, 2, 3}, + }) + if err != nil { + t.Fatal(err) + } + packet, err := DecodePacket(payload) + if err != nil { + t.Fatal(err) + } + frame, err := packet.VideoFrame() if err != nil { t.Fatal(err) } diff --git a/native/sender/internal/remote/peer.go b/native/sender/internal/remote/peer.go index e0f46cf..acfe31f 100644 --- a/native/sender/internal/remote/peer.go +++ b/native/sender/internal/remote/peer.go @@ -97,20 +97,37 @@ func newPeer(session *Session, peerID string, slot int, configuration webrtc.Con } } }) - sender, err := connection.AddTrack(session.fanout.Track()) + videoSender, err := connection.AddTrack(session.fanout.Track()) if err != nil { _ = connection.Close() return nil, errors.New("bind shared encoded track failed") } + var audioSender *webrtc.RTPSender + if session.audioFanout != nil { + audioSender, err = connection.AddTrack(session.audioFanout.Track()) + if err != nil { + _ = connection.Close() + return nil, errors.New("bind shared Opus track failed") + } + } if session.codec == media.CodecH264 { - transceivers := connection.GetTransceivers() - if len(transceivers) != 1 || len(session.codecPreferences) == 0 || - transceivers[0].SetCodecPreferences(session.codecPreferences) != nil { + var videoTransceiver *webrtc.RTPTransceiver + for _, transceiver := range connection.GetTransceivers() { + if transceiver.Kind() == webrtc.RTPCodecTypeVideo { + videoTransceiver = transceiver + break + } + } + if videoTransceiver == nil || len(session.codecPreferences) == 0 || + videoTransceiver.SetCodecPreferences(session.codecPreferences) != nil { _ = connection.Close() return nil, errors.New("configure H.264 codec preferences failed") } } - go peer.readRTCP(sender) + go peer.readRTCP(videoSender) + if audioSender != nil { + go peer.readRTCP(audioSender) + } return peer, nil } diff --git a/native/sender/internal/remote/session.go b/native/sender/internal/remote/session.go index af2d940..8738ed0 100644 --- a/native/sender/internal/remote/session.go +++ b/native/sender/internal/remote/session.go @@ -42,18 +42,20 @@ var ( ) type Event struct { - Kind string `json:"kind"` - Message string `json:"message,omitempty"` - ActiveViewers int `json:"activeViewers,omitempty"` - WaitingViewers int `json:"waitingViewers,omitempty"` - Media *media.Metrics `json:"media,omitempty"` - Peers []PeerDiagnostics `json:"peers,omitempty"` + Kind string `json:"kind"` + Message string `json:"message,omitempty"` + ActiveViewers int `json:"activeViewers,omitempty"` + WaitingViewers int `json:"waitingViewers,omitempty"` + Media *media.Metrics `json:"media,omitempty"` + Audio *media.AudioMetrics `json:"audio,omitempty"` + Peers []PeerDiagnostics `json:"peers,omitempty"` } type StartOptions struct { BaseURL *url.URL HostAdmissionPassword string Codec media.Codec + EnableAudio bool OnEvent func(Event) } @@ -82,6 +84,7 @@ type Session struct { peers map[string]*peer admission *viewerAdmission fanout *media.Fanout + audioFanout *media.AudioFanout codec media.Codec peerAPI *webrtc.API codecPreferences []webrtc.RTPCodecParameters @@ -185,6 +188,13 @@ func Start(parent context.Context, options StartOptions) (*Session, Room, error) cancel() return nil, Room{}, err } + if options.EnableAudio { + session.audioFanout, err = media.NewAudioFanout(func(fatalErr error) { session.fail(fatalErr) }) + if err != nil { + session.close() + return nil, Room{}, err + } + } session.peerAPI, session.codecPreferences, err = newPeerAPI(codec) if err != nil { session.close() @@ -228,18 +238,38 @@ func (session *Session) resetSignalingConnection() { } } -func (session *Session) WriteFrame(frame media.Frame) error { - if err := session.fanout.Push(frame); err != nil { - session.fail(fmt.Errorf("write encoded frame: %w", err)) - return err +func (session *Session) WriteMedia(packet media.Packet) error { + switch packet.Kind { + case media.KindVideo: + frame, err := packet.VideoFrame() + if err == nil { + err = session.fanout.Push(frame) + } + if err != nil { + session.fail(fmt.Errorf("write encoded video: %w", err)) + return err + } + return nil + case media.KindOpus: + if session.audioFanout == nil { + return errors.New("encoded audio was not enabled for this session") + } + if err := session.audioFanout.Push(packet); err != nil { + session.fail(fmt.Errorf("write encoded audio: %w", err)) + return err + } + return nil + default: + return errors.New("local sender supplied an unsupported encoded media kind") } - return nil } func (session *Session) Codec() media.Codec { return session.codec } +func (session *Session) AudioEnabled() bool { return session.audioFanout != nil } + func (session *Session) Stop() { session.close() } @@ -587,7 +617,12 @@ func (session *Session) diagnosticsLoop() { return diagnostics[left].Slot < diagnostics[right].Slot }) metrics := session.fanout.Snapshot() - session.emit(Event{Kind: "diagnostics", Media: &metrics, Peers: diagnostics}) + var audio *media.AudioMetrics + if session.audioFanout != nil { + value := session.audioFanout.Snapshot() + audio = &value + } + session.emit(Event{Kind: "diagnostics", Media: &metrics, Audio: audio, Peers: diagnostics}) } } } @@ -664,6 +699,9 @@ func (session *Session) close() { session.peers = make(map[string]*peer) session.mu.Unlock() session.fanout.Close() + if session.audioFanout != nil { + session.audioFanout.Close() + } for _, peer := range peers { peer.close() } diff --git a/native/sender/internal/windowaudio/provider.go b/native/sender/internal/windowaudio/provider.go new file mode 100644 index 0000000..0704e73 --- /dev/null +++ b/native/sender/internal/windowaudio/provider.go @@ -0,0 +1,25 @@ +package windowaudio + +import ( + "context" + "errors" +) + +var ErrUnavailable = errors.New("Windows 11 process audio is unavailable") + +type Target struct { + PID uint32 + CreationTime uint64 + Title string +} + +type PCMChunk struct { + Timestamp100ns uint64 + Duration100ns uint64 + Data []byte +} + +type Provider interface { + List(context.Context) ([]Target, error) + Capture(context.Context, Target, func(PCMChunk) error) error +} diff --git a/native/sender/internal/windowaudio/provider_other.go b/native/sender/internal/windowaudio/provider_other.go new file mode 100644 index 0000000..7fb74cf --- /dev/null +++ b/native/sender/internal/windowaudio/provider_other.go @@ -0,0 +1,15 @@ +//go:build !windows + +package windowaudio + +import "context" + +type unsupportedProvider struct{} + +func NewProvider() Provider { return unsupportedProvider{} } + +func (unsupportedProvider) List(context.Context) ([]Target, error) { return nil, ErrUnavailable } + +func (unsupportedProvider) Capture(context.Context, Target, func(PCMChunk) error) error { + return ErrUnavailable +} diff --git a/native/sender/internal/windowaudio/provider_windows.go b/native/sender/internal/windowaudio/provider_windows.go new file mode 100644 index 0000000..3fa6433 --- /dev/null +++ b/native/sender/internal/windowaudio/provider_windows.go @@ -0,0 +1,145 @@ +//go:build windows + +package windowaudio + +import ( + "context" + "encoding/binary" + "encoding/json" + "errors" + "io" + "os" + "os/exec" + "path/filepath" + "strconv" + "strings" + + "golang.org/x/sys/windows" +) + +const ( + maxListBytes = 64 << 10 + maxTargets = 100 + helperHeaderSize = 28 +) + +type windowsProvider struct{ helper string } + +type helperTarget struct { + PID uint32 `json:"pid"` + CreationTime uint64 `json:"creationTime"` + Title string `json:"title"` +} + +func NewProvider() Provider { + helper := strings.TrimSpace(os.Getenv("SCREENER_PROCESS_AUDIO_HELPER")) + if helper == "" { + if executable, err := os.Executable(); err == nil { + helper = filepath.Join(filepath.Dir(executable), "screener-process-audio.exe") + } + } + return &windowsProvider{helper: helper} +} + +func (provider *windowsProvider) available() bool { + version := windows.RtlGetVersion() + if version.MajorVersion < 10 || version.BuildNumber < 22_000 || provider.helper == "" { + return false + } + info, err := os.Stat(provider.helper) + return err == nil && !info.IsDir() +} + +func (provider *windowsProvider) List(ctx context.Context) ([]Target, error) { + if !provider.available() { + return nil, ErrUnavailable + } + command := exec.CommandContext(ctx, provider.helper, "--list") + stdout, err := command.StdoutPipe() + if err != nil { + return nil, ErrUnavailable + } + command.Stderr = io.Discard + if err = command.Start(); err != nil { + return nil, ErrUnavailable + } + payload, readErr := io.ReadAll(io.LimitReader(stdout, maxListBytes+1)) + if (readErr != nil || len(payload) > maxListBytes) && ctx.Err() == nil { + _ = command.Process.Kill() + } + waitErr := command.Wait() + if readErr != nil || waitErr != nil || len(payload) > maxListBytes { + return nil, ErrUnavailable + } + var listed []helperTarget + if json.Unmarshal(payload, &listed) != nil || len(listed) > maxTargets { + return nil, ErrUnavailable + } + targets := make([]Target, 0, len(listed)) + for _, candidate := range listed { + title := strings.TrimSpace(candidate.Title) + if candidate.PID == 0 || candidate.CreationTime == 0 || title == "" || len(title) > 512 { + continue + } + targets = append(targets, Target{ + PID: candidate.PID, CreationTime: candidate.CreationTime, Title: title, + }) + } + return targets, nil +} + +func (provider *windowsProvider) Capture(ctx context.Context, target Target, onChunk func(PCMChunk) error) error { + if !provider.available() || target.PID == 0 || target.CreationTime == 0 || onChunk == nil { + return ErrUnavailable + } + command := exec.CommandContext(ctx, provider.helper, "--capture", + strconv.FormatUint(uint64(target.PID), 10), strconv.FormatUint(target.CreationTime, 10)) + stdout, err := command.StdoutPipe() + if err != nil { + return ErrUnavailable + } + command.Stderr = io.Discard + if err = command.Start(); err != nil { + return ErrUnavailable + } + started := false + for { + header := make([]byte, helperHeaderSize) + _, readErr := io.ReadFull(stdout, header) + if readErr != nil { + err = readErr + break + } + length := binary.BigEndian.Uint32(header[24:28]) + if string(header[:4]) != "SPCM" || header[4] != 1 || header[5] != 0 || + header[6] != 0 || header[7] != 0 || length == 0 || length > 48_000*2*2/10 || length%4 != 0 { + err = errors.New("process audio helper returned an invalid PCM header") + break + } + data := make([]byte, length) + if _, readErr = io.ReadFull(stdout, data); readErr != nil { + err = readErr + break + } + started = true + if err = onChunk(PCMChunk{ + Timestamp100ns: binary.BigEndian.Uint64(header[8:16]), + Duration100ns: binary.BigEndian.Uint64(header[16:24]), + Data: data, + }); err != nil { + _ = command.Process.Kill() + break + } + } + if err != nil && ctx.Err() == nil { + _ = command.Process.Kill() + } + _ = command.Wait() + if ctx.Err() != nil { + return nil + } + if err != nil || !started { + return errors.New("target-process audio capture stopped") + } + return errors.New("target process exited") +} diff --git a/native/sender/internal/windowaudio/provider_windows_test.go b/native/sender/internal/windowaudio/provider_windows_test.go new file mode 100644 index 0000000..7c7a636 --- /dev/null +++ b/native/sender/internal/windowaudio/provider_windows_test.go @@ -0,0 +1,100 @@ +//go:build windows + +package windowaudio + +import ( + "context" + "encoding/binary" + "flag" + "os" + "testing" + "time" +) + +const testCreationTime = uint64(123456) + +var ( + captureHelperPID = flag.String("capture", "", "run as the process-audio capture test helper") + listHelper = flag.Bool("list", false, "run as the process-audio list test helper") +) + +func TestMain(main *testing.M) { + flag.Parse() + if *listHelper { + _, _ = os.Stdout.Write(make([]byte, maxListBytes+1)) + select {} + } + if *captureHelperPID != "" { + if len(flag.Args()) != 1 || flag.Args()[0] != "123456" { + os.Exit(3) + } + header := make([]byte, helperHeaderSize) + copy(header, "SPCM") + header[4] = 1 + binary.BigEndian.PutUint64(header[16:24], 200_000) + binary.BigEndian.PutUint32(header[24:28], 4) + _, _ = os.Stdout.Write(header) + _, _ = os.Stdout.Write(make([]byte, 4)) + _, _ = os.Stdout.Write(make([]byte, helperHeaderSize)) + select {} + } + os.Exit(main.Run()) +} + +func TestCapturePassesIdentityAndKillsHelperBeforeWaitingOnInvalidHeader(t *testing.T) { + provider := &windowsProvider{helper: os.Args[0]} + if !provider.available() { + t.Skip("requires Windows 11") + } + + ctx, cancel := context.WithCancel(context.Background()) + result := make(chan error, 1) + chunks := 0 + go func() { + result <- provider.Capture(ctx, Target{PID: 1, CreationTime: testCreationTime}, func(PCMChunk) error { + chunks++ + return nil + }) + }() + + select { + case err := <-result: + if err == nil { + t.Fatal("Capture returned nil after an invalid helper header") + } + if chunks != 1 { + t.Fatalf("Capture delivered %d chunks; process identity was not passed to the helper", chunks) + } + case <-time.After(2 * time.Second): + cancel() + <-result + t.Fatal("Capture waited on a running helper after an invalid header") + } + cancel() +} + +func TestListKillsHelperBeforeWaitingOnOversizeOutput(t *testing.T) { + provider := &windowsProvider{helper: os.Args[0]} + if !provider.available() { + t.Skip("requires Windows 11") + } + + ctx, cancel := context.WithCancel(context.Background()) + result := make(chan error, 1) + go func() { + _, err := provider.List(ctx) + result <- err + }() + + select { + case err := <-result: + if err == nil { + t.Fatal("List accepted oversized helper output") + } + case <-time.After(2 * time.Second): + cancel() + <-result + t.Fatal("List waited on a running helper after oversized output") + } + cancel() +} diff --git a/scripts/native-one-viewer-gate.ts b/scripts/native-one-viewer-gate.ts index 56f613b..8515965 100644 --- a/scripts/native-one-viewer-gate.ts +++ b/scripts/native-one-viewer-gate.ts @@ -119,6 +119,9 @@ interface ViewerSnapshot { videoHeight: number; videoCurrentTime: number; renderedFrames: number; + audioTrackCount: number; + audioPacketsReceived: number; + audioBytesReceived: number; } class CdpConnection { @@ -274,6 +277,7 @@ async function main(): Promise { const chromePath = requiredEnvironmentPath("SCREENER_NATIVE_GATE_CHROME"); const goPath = process.env.SCREENER_NATIVE_GATE_GO?.trim() || "go"; const gateCodec = process.env.SCREENER_NATIVE_GATE_CODEC?.trim() === "h264" ? "h264" : "vp8"; + const audioTargetTitle = process.env.SCREENER_NATIVE_GATE_AUDIO_TARGET_TITLE?.trim() || ""; const accessKey = randomBytes(9).toString("base64url"); const report: GateReport = { schemaVersion: 1, @@ -283,7 +287,9 @@ async function main(): Promise { limitations: [ "The source is a real animated Chrome tab captured through getDisplayMedia, not a game workload.", "This run proves one WebCodecs object, not one physical or hardware encoder.", - "The loopback run does not prove public STUN, restrictive networks, TURN, audio, packaging, or endurance.", + audioTargetTitle + ? "The loopback run does not prove public STUN, restrictive networks, TURN, packaging, or endurance." + : "The loopback run does not prove public STUN, restrictive networks, TURN, audio, packaging, or endurance.", ], }; let server: ScreenerServer | null = null; @@ -366,10 +372,22 @@ async function main(): Promise { await createPage(cdp, animatedSourceUrl()); const activeSenderPage = await createPage(cdp, launchUrl, senderProbe()); senderPage = activeSenderPage; + let audioTargetId = ""; + if (audioTargetTitle) { + await waitForSample( + (deadline) => evaluate(cdp!, activeSenderPage, + `([...document.querySelector('#audio-target').options].find((option) => option.text.includes(${JSON.stringify(audioTargetTitle)}))?.value || '')`, deadline), + Boolean, + 5_000, + ); + audioTargetId = await evaluate(cdp, activeSenderPage, + `([...document.querySelector('#audio-target').options].find((option) => option.text.includes(${JSON.stringify(audioTargetTitle)}))?.value || '')`, Date.now() + 2_000); + } await evaluate(cdp, activeSenderPage, `(() => { document.querySelector('#server-url').value = ${JSON.stringify(baseUrl)}; document.querySelector('#password').value = ${JSON.stringify(accessKey)}; document.querySelector('#codec').value = ${JSON.stringify(gateCodec)}; + document.querySelector('#audio-target').value = ${JSON.stringify(audioTargetId)}; document.querySelector('#start').click(); })()`, Date.now() + 5_000); await waitForSenderStart( @@ -464,6 +482,10 @@ async function main(): Promise { sameBridge: retainsFirstBridgeSend(sender), oneEncoderObject: sender.encoderInstances === 1, criticalErrors: sender.fatalEvents === 0 && sender.encoderErrors === 0, + ...(audioTargetTitle ? { + audioTrack: viewer.audioTrackCount === 1, + audioInbound: viewer.audioPacketsReceived > beforeViewer.audioPacketsReceived, + } : {}), }; report.stages["viewer-media"] = { ...checks, @@ -471,6 +493,7 @@ async function main(): Promise { byteDelta: viewer.bytesReceived - beforeViewer.bytesReceived, decodedDelta: viewer.framesDecoded - beforeViewer.framesDecoded, renderedDelta: viewer.renderedFrames - beforeViewer.renderedFrames, + audioPacketDelta: viewer.audioPacketsReceived - beforeViewer.audioPacketsReceived, pionPacketDelta: (sender.peers[0]?.packetsSent ?? 0) - (beforeSender.peers[0]?.packetsSent ?? 0), pcOrdinal: viewer.identity.pcOrdinal, pionSlot: sender.peers[0]?.slot ?? -1, @@ -648,6 +671,7 @@ function viewerProbe(): string { consistent: true, pendingOffer: null, pcs: [], boundPcOrdinal: 0, boundConnectionOrdinal: 0, boundSocketOrdinal: 0, boundAuthGeneration: 0, trackCount: 0, trackOrdinal: 0, trackPcOrdinal: 0, + audioTrackCount: 0, renderedFrames: 0, observedVideo: null }; const connectionOrdinal = (value) => { let index = state.connectionIds.indexOf(value); @@ -733,6 +757,7 @@ function viewerProbe(): string { state.pcs.push(this); this.__gatePcOrdinal = state.pcs.length; this.addEventListener('track', (event) => { + if (event.track?.kind === 'audio') { state.audioTrackCount += 1; return; } if (event.track?.kind !== 'video') return; state.trackCount += 1; state.trackOrdinal = state.trackCount; state.trackPcOrdinal = this.__gatePcOrdinal; @@ -762,6 +787,7 @@ function viewerProbe(): string { } const pc = state.pcs[state.boundPcOrdinal - 1]; let packetsReceived = 0, bytesReceived = 0, framesDecoded = 0; + let audioPacketsReceived = 0, audioBytesReceived = 0; if (pc) { const stats = await pc.getStats(); stats.forEach((entry) => { @@ -770,6 +796,10 @@ function viewerProbe(): string { bytesReceived += finite(entry.bytesReceived); framesDecoded += finite(entry.framesDecoded); } + if (entry.type === 'inbound-rtp' && entry.kind === 'audio' && entry.isRemote !== true) { + audioPacketsReceived += finite(entry.packetsReceived); + audioBytesReceived += finite(entry.bytesReceived); + } }); } const activeOrdinal = state.activeConnectionId === null ? 0 : @@ -787,7 +817,8 @@ function viewerProbe(): string { iceConnectionState: pc?.iceConnectionState || 'none', packetsReceived, bytesReceived, framesDecoded, videoReadyState: finite(video?.readyState), videoWidth: finite(video?.videoWidth), videoHeight: finite(video?.videoHeight), videoCurrentTime: finite(video?.currentTime), - renderedFrames: state.renderedFrames }; + renderedFrames: state.renderedFrames, audioTrackCount: state.audioTrackCount, + audioPacketsReceived, audioBytesReceived }; } const finite = (value) => Number.isFinite(value) ? value : 0; Object.defineProperty(globalThis, '__NATIVE_VIEWER_GATE__', {