From cfc323adce4f8bf136a8417fb2cef3944857562a Mon Sep 17 00:00:00 2001 From: AAA <35992542+TNTcraftHIM@users.noreply.github.com> Date: Wed, 9 Sep 2026 06:09:06 +0800 Subject: [PATCH] feat(media): integrate embedded media services and shared native encoding Embed STUN and SFU in the shared Go process, reuse native encoding pipelines with independent adaptation, and complete Client lifecycle, settings, diagnostics and packaging fixes. Retain standard Browser senders after measured encoded-reuse limitations. --- .env.example | 32 +- AGENTS.md | 3 +- CONTRIBUTING.md | 11 +- README.md | 66 +- README.zh-CN.md | 92 + cmd/screener-client/README.md | 137 +- cmd/screener-client/main.go | 2 + cmd/screener-peer-gate/main.go | 8 +- cmd/screener-server/diagnostics_unix.go | 33 + cmd/screener-server/diagnostics_windows.go | 6 + cmd/screener-server/main.go | 64 +- cmd/screener-server/main_test.go | 24 + deploy/container/Dockerfile | 8 + deploy/coturn/turnserver.conf.example | 33 - deploy/livekit/livekit.yaml.example | 43 - deploy/nginx/share.bonfire.icu.conf.example | 43 +- deploy/release-app.sh | 11 +- deploy/systemd/coturn.service.example | 39 - deploy/systemd/livekit.service.example | 39 - .../screener-livekit-readiness.conf.example | 9 - deploy/systemd/screener.service.example | 5 +- .../0005-automatic-hybrid-media-routing.md | 6 + ...07-path-isolated-representation-quality.md | 6 + .../adr/0010-cross-platform-client-runtime.md | 22 +- .../0011-browser-assisted-native-fanout.md | 6 +- docs/adr/0013-embedded-node-local-media.md | 122 + docs/deployment.md | 53 +- docs/operations/self-hosting.md | 233 +- docs/product/media-quality.md | 104 +- docs/product/presentation-lifecycle.md | 13 +- docs/product/rooms-access.md | 21 +- docs/product/routing-transport.md | 49 +- docs/project-memory.md | 62 +- docs/reference/configuration.md | 112 +- docs/research/advanced-peer-distribution.md | 146 +- .../browser-encoded-fanout-continuation.json | 368 ++ .../research/data/browser-encoded-fanout.json | 287 ++ docs/research/data/node-local-browser.json | 1 + docs/research/data/node-local-feedback.json | 1 + docs/research/data/node-local-group.json | 506 +++ docs/research/data/node-local-vp8.json | 3354 +++++++++++++++++ docs/research/data/webrtc-encoder-chain.json | 111 + .../research/data/webrtc-encoder-latency.json | 597 +++ docs/research/data/webrtc-encoder-pool.json | 861 +++++ docs/research/data/webrtc-h264-pipeline.json | 44 + docs/research/embedded-media.md | 112 + docs/research/native-client-lifecycle.md | 133 +- docs/research/node-local-adaptation.md | 217 ++ docs/research/node-local-browser-probe.md | 106 + docs/research/node-local-encoding-probe.md | 124 + docs/research/node-local-media-design.md | 474 +++ docs/research/webrtc-encoder-pool.md | 663 ++++ docs/status.md | 245 +- docs/todo.md | 254 +- docs/verification-status.md | 5 +- go.mod | 78 +- go.sum | 327 +- internal/client/clientapp/app.go | 119 +- internal/client/clientapp/app_test.go | 97 + internal/client/clientapp/console.go | 125 +- internal/client/clientapp/console_test.go | 74 + internal/client/loopback/protocol.go | 4 +- internal/client/loopback/protocol_test.go | 18 +- internal/client/loopback/server.go | 114 +- internal/client/loopback/server_test.go | 98 +- internal/client/mediaedge/audio.go | 23 +- internal/client/mediaedge/edge.go | 202 +- internal/client/mediaedge/edge_debug_test.go | 65 + internal/client/mediaedge/edge_test.go | 254 +- internal/client/mediaedge/engine.go | 149 +- .../client/mediaedge/group_fixture_test.go | 241 ++ internal/client/mediaedge/groups.go | 252 ++ internal/client/mediaedge/groups_test.go | 364 ++ internal/client/mediaedge/publication.go | 228 ++ internal/client/mediaedge/publication_test.go | 178 + internal/client/mediaedge/quality.go | 115 +- internal/client/mediaedge/quality_test.go | 110 +- internal/client/mediaedge/receiver.go | 41 +- internal/client/mediaedge/receiver_test.go | 65 +- .../mediaedge/relay_chain_fixture_test.go | 299 ++ internal/client/mediaedge/relay_derivation.go | 422 +++ .../client/mediaedge/relay_fixture_test.go | 483 +++ internal/client/mediaedge/relay_input.go | 128 + internal/client/mediaedge/relay_input_test.go | 213 ++ .../mediaedge/relay_network_fixture_test.go | 353 ++ internal/client/mediaedge/source.go | 635 +++- internal/client/mediaedge/source_test.go | 112 + .../mediaedge/testdata/gcc_feedback_probe.go | 204 + internal/client/nativeaudio/encoder.go | 2 + .../nativecapture/adaptive_fixture_test.go | 159 + internal/client/nativecapture/probe.go | 2 +- internal/client/nativecapture/probe_test.go | 8 +- internal/client/nativecapture/protocol.go | 148 +- .../client/nativecapture/protocol_test.go | 183 +- internal/client/nativecapture/stream.go | 262 +- internal/client/nativecapture/stream_test.go | 79 + internal/client/nativecontrol/publication.go | 83 + internal/client/nativecontrol/session.go | 166 +- internal/client/nativecontrol/session_test.go | 283 +- internal/client/nativecontrol/wire.go | 56 +- internal/client/nativehost/audio_test.go | 180 + internal/client/nativehost/publication.go | 79 + .../client/nativehost/publication_test.go | 52 + internal/client/nativehost/session.go | 436 ++- internal/client/nativehost/session_test.go | 260 +- internal/client/nativeviewer/session.go | 40 +- internal/client/nativeviewer/session_test.go | 28 + internal/diagnostics/recorder.go | 271 ++ internal/diagnostics/recorder_test.go | 184 + internal/media/encoded/packetizer.go | 87 + internal/media/encoded/packetizer_test.go | 84 + internal/media/forwarding/encoded_source.go | 125 + .../media/forwarding/encoded_source_test.go | 68 + internal/media/forwarding/output.go | 389 ++ internal/media/forwarding/output_test.go | 180 + internal/media/forwarding/publication.go | 636 ++++ internal/media/forwarding/publication_test.go | 263 ++ internal/media/forwarding/recovery_test.go | 179 + internal/media/forwarding/retarget_test.go | 202 + internal/media/forwarding/source.go | 308 ++ internal/media/forwarding/source_test.go | 121 + internal/media/forwarding/transport.go | 635 ++++ internal/media/forwarding/transport_test.go | 438 +++ internal/server/app/access.go | 22 + internal/server/app/app.go | 170 +- internal/server/app/app_test.go | 467 ++- internal/server/app/routes.go | 43 +- internal/server/config/config.go | 116 +- internal/server/config/config_test.go | 196 +- internal/server/config/ice.go | 19 + internal/server/config/local_test.go | 13 +- internal/server/protocol/client.go | 2 + internal/server/protocol/constants.go | 3 +- internal/server/protocol/contract_test.go | 23 +- internal/server/protocol/fixture_test.go | 1 - internal/server/protocol/scalars.go | 13 - internal/server/protocol/scalars_test.go | 4 - internal/server/protocol/server.go | 41 +- internal/server/protocol/sfu.go | 115 + internal/server/route/api.go | 1 + internal/server/route/debug.go | 16 +- internal/server/route/debug_test.go | 24 + internal/server/route/operation.go | 6 +- internal/server/sfu/admission.go | 117 +- internal/server/sfu/admission_test.go | 80 +- internal/server/sfu/media.go | 762 ++++ internal/server/sfu/media_test.go | 361 ++ internal/server/sfu/roomcontrol.go | 189 - internal/server/sfu/roomcontrol_test.go | 438 --- internal/server/sfu/roomservice.go | 188 - internal/server/sfu/roomservice_test.go | 178 - internal/server/sfu/sfutest/fake.go | 336 +- internal/server/sfu/sfutest/fake_test.go | 193 - internal/server/sfu/token.go | 203 - internal/server/sfu/token_test.go | 225 -- internal/server/signal/router.go | 141 +- internal/server/signal/router_pump.go | 221 +- internal/server/signal/router_sfu.go | 394 +- internal/server/signal/router_sfu_signal.go | 254 ++ .../server/signal/router_sfu_signal_test.go | 299 ++ internal/server/signal/router_test.go | 539 +-- internal/server/signal/server.go | 51 +- internal/server/signal/server_test.go | 46 +- internal/server/signal/session.go | 24 +- internal/server/stun/server.go | 54 + internal/server/stun/server_test.go | 99 + licenses/README.md | 20 +- licenses/libvpx-1.17.0.txt | 327 -- licenses/upstream.json | 8 +- licenses/webrtc-m152.7977.0.2.txt | 2090 ++++++++++ native/capture/darwin/main.swift | 915 ++++- native/capture/linux/main.c | 691 +++- native/capture/linux/main.test.c | 118 + native/capture/windows/README.md | 17 +- native/capture/windows/adaptive_encoder.cpp | 600 +++ native/capture/windows/adaptive_encoder.h | 38 + native/capture/windows/build-libvpx.ps1 | 68 - native/capture/windows/build.ps1 | 74 +- native/capture/windows/capture_control.h | 58 + .../capture/windows/capture_control.test.cpp | 244 ++ native/capture/windows/capture_input.h | 101 + native/capture/windows/capture_output.h | 88 + .../capture/windows/encoded_group.fixture.cpp | 87 + .../windows/encoded_variants.probe.cpp | 276 ++ native/capture/windows/get-webrtc.ps1 | 63 + native/capture/windows/h264_decoder.h | 171 + native/capture/windows/h264_encoder.cpp | 746 ++++ native/capture/windows/h264_encoder.h | 253 ++ .../capture/windows/libvpx-dependencies.json | 17 - native/capture/windows/main.cpp | 1792 +++------ native/capture/windows/output_mailbox.h | 108 + native/capture/windows/vp8_decoder.h | 75 + native/capture/windows/vp8_encoder.cpp | 12 +- native/capture/windows/vp8_encoder.h | 3 + .../capture/windows/webrtc-dependencies.json | 19 + native/fixtures/mf-h264-hardware/build.ps1 | 8 +- native/probes/encoder-pool/CMakeLists.txt | 27 + native/probes/encoder-pool/README.md | 99 + .../encoder-pool/h264_encoder_adapter.cpp | 310 ++ .../encoder-pool/h264_encoder_adapter.h | 13 + native/probes/encoder-pool/harness.cpp | 567 +++ native/probes/encoder-pool/pool.h | 316 ++ package-lock.json | 194 - package.json | 4 +- scripts/assemble-client.mjs | 35 +- scripts/browser-encoded-fanout-probe.mjs | 143 + scripts/build-workspace.mjs | 21 + scripts/check-client.mjs | 51 +- scripts/client-browser-fanout-gate.ts | 1 + scripts/client-media-gate.ts | 55 +- scripts/client-native-host-gate.ts | 145 +- scripts/client-native-sfu-gate.ts | 388 -- scripts/client-native-sfu-page.ts | 186 - scripts/client-package-targets.mjs | 12 + scripts/client-package-targets.test.mjs | 25 + scripts/embedded-sfu-gate.ts | 303 ++ scripts/embedded-sfu-page.ts | 315 ++ scripts/encoded-group-decode.mjs | 83 + scripts/encoded-variants-browser.mjs | 47 + scripts/encoded-variants-probe.mjs | 65 + scripts/encoded-variants-worker.js | 221 ++ scripts/package-app-release.mjs | 5 +- scripts/package-client-candidate.mjs | 9 +- scripts/package-licenses.mjs | 9 +- scripts/required-project-paths.txt | 1 - src/client/App.tsx | 2 + src/client/components/living/primitives.tsx | 10 +- src/client/lib/debug.ts | 168 + src/client/lib/display-name.ts | 17 +- src/client/lib/session.ts | 187 +- src/client/lib/signaling.ts | 14 + src/client/media/host-sfu-route.ts | 38 +- src/client/media/quality.ts | 22 +- src/client/media/sender-quality-evidence.ts | 20 +- src/client/media/sfu-standby-prewarmer.ts | 65 - src/client/media/viewer-sfu-route.ts | 104 +- src/client/native/client.ts | 220 +- src/client/native/media-bridge.ts | 72 +- src/client/native/media-ingress.ts | 2 + src/client/native/native-sender-edge.ts | 3 +- src/client/native/native-sender-peer.ts | 29 +- src/client/native/native-sfu-publisher.ts | 220 ++ src/client/native/native-viewer-peer.ts | 16 +- src/client/native/wire.ts | 134 +- src/client/pages/HostPage.tsx | 171 +- src/client/pages/ViewerPage.tsx | 44 +- src/client/pages/host-page-notices.ts | 2 + src/client/sfu/connection-options.ts | 8 - src/client/sfu/peer.ts | 157 + src/client/sfu/publisher.ts | 1771 +++------ src/client/sfu/subscriber.ts | 768 +--- src/client/types.ts | 18 +- src/client/webrtc/host-peer.ts | 22 +- src/shared/protocol.ts | 60 +- tests/browser-debug.test.ts | 88 + tests/client-state.test.ts | 222 +- tests/fixtures/wire-samples.json | 97 +- tests/host-page-notice.test.ts | 6 + tests/host-peer-tracks.test.ts | 34 + tests/host-quality-ownership.test.ts | 197 + tests/native-client.test.ts | 41 +- tests/native-media-bridge.test.ts | 70 +- tests/native-media-ingress.test.ts | 2 +- tests/native-sender-edge.test.ts | 7 +- tests/native-sender-peer.test.ts | 32 +- tests/native-sfu-publisher.test.ts | 148 + tests/native-viewer-peer.test.ts | 81 +- tests/protocol-fixture.test.ts | 4 +- tests/protocol.test.ts | 23 +- tests/route-transition.test.ts | 136 +- tests/sender-quality-evidence.test.ts | 25 +- tests/sfu-client.test.ts | 2980 +++------------ tests/sfu-standby-prewarmer.test.ts | 115 - tsconfig.tools.json | 4 +- 274 files changed, 37854 insertions(+), 13238 deletions(-) create mode 100644 README.zh-CN.md create mode 100644 cmd/screener-server/diagnostics_unix.go create mode 100644 cmd/screener-server/diagnostics_windows.go create mode 100644 deploy/container/Dockerfile delete mode 100644 deploy/coturn/turnserver.conf.example delete mode 100644 deploy/livekit/livekit.yaml.example delete mode 100644 deploy/systemd/coturn.service.example delete mode 100644 deploy/systemd/livekit.service.example delete mode 100644 deploy/systemd/screener-livekit-readiness.conf.example create mode 100644 docs/adr/0013-embedded-node-local-media.md create mode 100644 docs/research/data/browser-encoded-fanout-continuation.json create mode 100644 docs/research/data/browser-encoded-fanout.json create mode 100644 docs/research/data/node-local-browser.json create mode 100644 docs/research/data/node-local-feedback.json create mode 100644 docs/research/data/node-local-group.json create mode 100644 docs/research/data/node-local-vp8.json create mode 100644 docs/research/data/webrtc-encoder-chain.json create mode 100644 docs/research/data/webrtc-encoder-latency.json create mode 100644 docs/research/data/webrtc-encoder-pool.json create mode 100644 docs/research/data/webrtc-h264-pipeline.json create mode 100644 docs/research/embedded-media.md create mode 100644 docs/research/node-local-adaptation.md create mode 100644 docs/research/node-local-browser-probe.md create mode 100644 docs/research/node-local-encoding-probe.md create mode 100644 docs/research/node-local-media-design.md create mode 100644 docs/research/webrtc-encoder-pool.md create mode 100644 internal/client/mediaedge/edge_debug_test.go create mode 100644 internal/client/mediaedge/group_fixture_test.go create mode 100644 internal/client/mediaedge/groups.go create mode 100644 internal/client/mediaedge/groups_test.go create mode 100644 internal/client/mediaedge/publication.go create mode 100644 internal/client/mediaedge/publication_test.go create mode 100644 internal/client/mediaedge/relay_chain_fixture_test.go create mode 100644 internal/client/mediaedge/relay_derivation.go create mode 100644 internal/client/mediaedge/relay_fixture_test.go create mode 100644 internal/client/mediaedge/relay_input.go create mode 100644 internal/client/mediaedge/relay_input_test.go create mode 100644 internal/client/mediaedge/relay_network_fixture_test.go create mode 100644 internal/client/mediaedge/source_test.go create mode 100644 internal/client/mediaedge/testdata/gcc_feedback_probe.go create mode 100644 internal/client/nativecapture/adaptive_fixture_test.go create mode 100644 internal/client/nativecapture/stream_test.go create mode 100644 internal/client/nativecontrol/publication.go create mode 100644 internal/client/nativehost/audio_test.go create mode 100644 internal/client/nativehost/publication.go create mode 100644 internal/client/nativehost/publication_test.go create mode 100644 internal/client/nativeviewer/session_test.go create mode 100644 internal/diagnostics/recorder.go create mode 100644 internal/diagnostics/recorder_test.go create mode 100644 internal/media/encoded/packetizer.go create mode 100644 internal/media/encoded/packetizer_test.go create mode 100644 internal/media/forwarding/encoded_source.go create mode 100644 internal/media/forwarding/encoded_source_test.go create mode 100644 internal/media/forwarding/output.go create mode 100644 internal/media/forwarding/output_test.go create mode 100644 internal/media/forwarding/publication.go create mode 100644 internal/media/forwarding/publication_test.go create mode 100644 internal/media/forwarding/recovery_test.go create mode 100644 internal/media/forwarding/retarget_test.go create mode 100644 internal/media/forwarding/source.go create mode 100644 internal/media/forwarding/source_test.go create mode 100644 internal/media/forwarding/transport.go create mode 100644 internal/media/forwarding/transport_test.go create mode 100644 internal/server/protocol/sfu.go create mode 100644 internal/server/route/debug_test.go create mode 100644 internal/server/sfu/media.go create mode 100644 internal/server/sfu/media_test.go delete mode 100644 internal/server/sfu/roomcontrol.go delete mode 100644 internal/server/sfu/roomcontrol_test.go delete mode 100644 internal/server/sfu/roomservice.go delete mode 100644 internal/server/sfu/roomservice_test.go delete mode 100644 internal/server/sfu/sfutest/fake_test.go delete mode 100644 internal/server/sfu/token.go delete mode 100644 internal/server/sfu/token_test.go create mode 100644 internal/server/signal/router_sfu_signal.go create mode 100644 internal/server/signal/router_sfu_signal_test.go create mode 100644 internal/server/stun/server.go create mode 100644 internal/server/stun/server_test.go delete mode 100644 licenses/libvpx-1.17.0.txt create mode 100644 licenses/webrtc-m152.7977.0.2.txt create mode 100644 native/capture/linux/main.test.c create mode 100644 native/capture/windows/adaptive_encoder.cpp create mode 100644 native/capture/windows/adaptive_encoder.h delete mode 100644 native/capture/windows/build-libvpx.ps1 create mode 100644 native/capture/windows/capture_control.h create mode 100644 native/capture/windows/capture_control.test.cpp create mode 100644 native/capture/windows/capture_input.h create mode 100644 native/capture/windows/capture_output.h create mode 100644 native/capture/windows/encoded_group.fixture.cpp create mode 100644 native/capture/windows/encoded_variants.probe.cpp create mode 100644 native/capture/windows/get-webrtc.ps1 create mode 100644 native/capture/windows/h264_decoder.h create mode 100644 native/capture/windows/h264_encoder.cpp create mode 100644 native/capture/windows/h264_encoder.h delete mode 100644 native/capture/windows/libvpx-dependencies.json create mode 100644 native/capture/windows/output_mailbox.h create mode 100644 native/capture/windows/vp8_decoder.h create mode 100644 native/capture/windows/webrtc-dependencies.json create mode 100644 native/probes/encoder-pool/CMakeLists.txt create mode 100644 native/probes/encoder-pool/README.md create mode 100644 native/probes/encoder-pool/h264_encoder_adapter.cpp create mode 100644 native/probes/encoder-pool/h264_encoder_adapter.h create mode 100644 native/probes/encoder-pool/harness.cpp create mode 100644 native/probes/encoder-pool/pool.h create mode 100644 scripts/browser-encoded-fanout-probe.mjs create mode 100644 scripts/build-workspace.mjs delete mode 100644 scripts/client-native-sfu-gate.ts delete mode 100644 scripts/client-native-sfu-page.ts create mode 100644 scripts/client-package-targets.test.mjs create mode 100644 scripts/embedded-sfu-gate.ts create mode 100644 scripts/embedded-sfu-page.ts create mode 100644 scripts/encoded-group-decode.mjs create mode 100644 scripts/encoded-variants-browser.mjs create mode 100644 scripts/encoded-variants-probe.mjs create mode 100644 scripts/encoded-variants-worker.js create mode 100644 src/client/lib/debug.ts delete mode 100644 src/client/media/sfu-standby-prewarmer.ts create mode 100644 src/client/native/native-sfu-publisher.ts delete mode 100644 src/client/sfu/connection-options.ts create mode 100644 src/client/sfu/peer.ts create mode 100644 tests/browser-debug.test.ts create mode 100644 tests/host-quality-ownership.test.ts create mode 100644 tests/native-sfu-publisher.test.ts delete mode 100644 tests/sfu-standby-prewarmer.test.ts diff --git a/.env.example b/.env.example index 3a1220a..a5bdc19 100644 --- a/.env.example +++ b/.env.example @@ -24,23 +24,29 @@ ENDPOINT_MEDIA_COPY_CAPACITY=2 # During pre-release route canaries, this enables sanitized room/ordinal events. # SCREENER_DEBUG=route -# Optional automatic SFU fallback. Supply all four values together; leaving them -# empty keeps LiveKit completely dormant. +# Optional in-process automatic SFU fallback. Set SFU_UDP_PORT=7882 to enable; +# unset or empty keeps it disabled. Local Client modes never create this listener. # Ingress uses the 9000-room code capacity; egress uses that capacity times the # configured per-room Viewer admission. -# Production requires wss and an independent API secret of 32+ bytes. -LIVEKIT_URL= -LIVEKIT_API_URL= -LIVEKIT_API_KEY= -LIVEKIT_API_SECRET= +SFU_UDP_PORT= +# IPv4 bind address, independent of HTTP LISTEN_HOST and advertised media IP. +SFU_LISTEN_HOST=0.0.0.0 +# Optional advertised IPv4 override when the listener's address is behind NAT. +SFU_PUBLIC_IP= # Ordinary peer ICE discovery. Production requires at least one STUN URL. -# Separate multiple URLs with commas. +# These are advertised addresses, not local listener addresses. They may use +# an unproxied DNS name distinct from PUBLIC_BASE_URL. Separate URLs with commas. STUN_URLS= +# Hosted STUN is in-process. With STUN_URLS configured, bind this IPv4 address +# on UDP 3478 (also 3479/3480 below). It is independent of HTTP LISTEN_HOST. +# Local Client modes never create these listeners, even with public discovery. +STUN_LISTEN_HOST=0.0.0.0 # Optional bounded NAT traversal support. When enabled, Hosts see a per-share -# switch that defaults on, and Browser P2P derives self-hosted UDP 3479/3480 -# from the first STUN_URLS endpoint on UDP 3478. +# switch that defaults on after all three local STUN listeners bind. Browser +# P2P derives UDP 3479/3480 from the first STUN_URLS endpoint on UDP 3478. +# Cloud security-group and host firewall rules must also allow these UDP ports. NAT_PREDICTION_ENABLED=false # Development example (two processes): Vite serves the Browser on 8787 and @@ -55,8 +61,6 @@ NAT_PREDICTION_ENABLED=false # STUN_URLS=stun:stun.example.com:3478 # NAT_PREDICTION_ENABLED=true # Automatic SFU fallback (does not replace STUN above): -# LIVEKIT_URL=wss://share.example.com -# LIVEKIT_API_URL=http://127.0.0.1:7880 -# LIVEKIT_API_KEY=replace-with-a-livekit-api-key -# LIVEKIT_API_SECRET=replace-with-at-least-32-independent-random-bytes +# SFU_UDP_PORT=7882 +# SFU_PUBLIC_IP=198.51.100.5 # ENDPOINT_MEDIA_COPY_CAPACITY=2 diff --git a/AGENTS.md b/AGENTS.md index 0de9621..b8220c2 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,7 +31,8 @@ [media quality](./docs/product/media-quality.md), and [presentation/lifecycle](./docs/product/presentation-lifecycle.md). - Media stays automatic and P2P-first; central services provide room authority, - signaling, STUN, observability, and bounded LiveKit SFU/UDP fallback. Route + signaling, STUN, observability, and bounded embedded SFU/UDP fallback using + mature LiveKit media components under [ADR-0013](./docs/adr/0013-embedded-node-local-media.md). Route changes must preserve the single-graph, single-operation model in [ADR-0005](./docs/adr/0005-automatic-hybrid-media-routing.md). - WebRTC and LiveKit own network and media adaptation. Do not add custom quality diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index aa2ea1f..c32205f 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -71,9 +71,14 @@ Clean up worktrees and branches only after semantic review and integration are c isolated profiles, bounded deadlines, shared cleanup helpers, and a structured result. A manual diagnostic may locate a failure, but is not retained as pass evidence. -- A validated push to `main` uploads a short-lived Server candidate. The three - platform Client candidates run only from an explicit manual workflow dispatch - with `client_checks=true`, because they are the expensive part of the matrix. +- On Windows, execute Client/Server and Go test binaries only from stable + project build paths. Go's temporary compiler/cache files are not firewall + identities; do not run network tests with a changing temporary executable + path. Keep browser profiles isolated and consider the active firewall when + diagnosing connection failures rather than changing global firewall rules. +- Server and three-platform Client candidates run only from an explicit manual + workflow dispatch with `client_checks=true`. Ordinary `main` pushes validate + source without running the expensive packaging matrix. Publishing a tag or GitHub Release remains an explicit release decision rather than a side effect of every merge. diff --git a/README.md b/README.md index 535a6ae..ff9b88d 100644 --- a/README.md +++ b/README.md @@ -1,22 +1,37 @@ # Screener +English | [简体中文](./README.zh-CN.md) + Screener is a private, low-latency screen-sharing tool for one game player and up to 20 authenticated friends. Web Viewers need no installation. Media is -automatic and P2P-first, with bounded Browser relay and one dedicated LiveKit -SFU/UDP fallback rather than an always-SFU conference topology. +automatic and P2P-first, with bounded Browser relay and one embedded SFU/UDP +fallback. The Hosted Go process owns signaling, STUN, and admitted SFU media. -[Project memory](./docs/project-memory.md) is the compact product map. -[Current status](./docs/status.md) owns the exact source and production state; -this README intentionally does not mirror release details. +Windows Client and Browser are the current acceptance targets. macOS and Linux +have source/build support; their physical acceptance is separate. See +[current status](./docs/status.md) for release readiness and the distinction +between current source and the last recorded production deployment. ## Screener Client -The Client opens the same interface in your system browser. The package is one -self-contained executable. +For a matching Client bundle: + +1. Extract the whole package, keeping its `runtime` directory beside the executable. +2. Run `screener-client.exe` on Windows, `./screener-client` on Linux, or + `Screener Client.app` on macOS. +3. Choose a mode, select a source, and send the room invitation to your friends. + +The Client opens the interface in your system browser; it has no embedded +browser UI. The package carries the Go application and native capture/public-link +helpers. Running the package does not require Node.js, npm, or Go. Linux native +capture uses system Portal/PipeWire/GStreamer dependencies; see the +[Client guide](./cmd/screener-client/README.md). - **Local:** share with devices on the same LAN. - **Public link:** send an ordinary invitation link. Cloudflare Quick Tunnel carries room access and signaling; media travels between peers without an SFU. + The link lasts for that Client run. Relaunching creates a new temporary link; + Quick Tunnel provides no uptime guarantee. - **Site:** open a saved Screener server and use the running Client for native capture or compatible media forwarding. The Site can provide its SFU fallback. @@ -26,11 +41,36 @@ application when supported. Audio has a separate sharing switch. Native capture on Windows offers VP8, Auto, and H264 in the same controls as Browser capture. Other native platform adapters currently use hardware H.264. -P2P is best effort, not a guarantee that every network pair connects. The -[current status](./docs/status.md) lists platform acceptance and open Client -issues. Client access is open by default; its launcher can set an optional +Native parents reuse encoded outputs for compatible direct children and derive +a missing lower output only when needed. Each endpoint has a bounded outgoing-copy +capacity, default two; incompatible weak paths can use separate local outputs. +Native SFU publication uses that same source. Ordinary Browser capture and relay +retain WebRTC senders; encoded reuse is a Native capability. See +[media quality](./docs/product/media-quality.md) for the behavior and bounds. + +Media requires a working UDP path; P2P does not guarantee connectivity between +every network pair. Browser/OS suspension and network policy can interrupt a +share. Client access is open by default; its launcher can set an optional site-access password. This does not change access rules on a saved Site. +If a Chromium-based Browser cannot establish media connections, see the +[Chromium WebRTC FAQ](./cmd/screener-client/README.md#chromium-webrtc-connections). + +For feedback, start the Client with `--debug`, reproduce the problem, and press +`D` in its terminal to export a local diagnostic ZIP. Browser diagnostics are +separate; see [diagnostics and export](./docs/reference/configuration.md#diagnostics). + +## Self-Hosting + +The server package runs `screener-server` with the Browser assets embedded in +the binary. Initial setup, configuration and coordinated embedded-media cutover +are documented in [self-hosting](./docs/operations/self-hosting.md). +[Deployment and recovery](./docs/deployment.md) covers matching artifacts and +later application updates. These instructions do not imply that a public +GitHub Release or Docker image has already been published. +An [optional container recipe](./docs/operations/self-hosting.md#optional-container) +uses the same verified Server release. + ## Local Development Node.js 24, npm 11, and Go 1.26 are required. @@ -40,7 +80,9 @@ npm ci npm run dev ``` -In a second terminal, run the application server that the dev server proxies: +In a second terminal, run the application server that the dev server proxies. +The command below uses POSIX environment syntax; PowerShell uses the corresponding +`$env:` variables. ```sh PORT=8788 PUBLIC_BASE_URL=http://localhost:8787 go run ./cmd/screener-server @@ -78,3 +120,5 @@ npm run check:client Screener's own code is [MIT licensed](./LICENSE). Redistributed components retain their own licenses and notices; see [third-party licensing](./licenses/README.md). +The software is provided "AS IS", without warranty; performance and connectivity +depend on the devices, Browsers and networks in use. diff --git a/README.zh-CN.md b/README.zh-CN.md new file mode 100644 index 0000000..2ff536f --- /dev/null +++ b/README.zh-CN.md @@ -0,0 +1,92 @@ +# Screener + +[English](./README.md) | 简体中文 + +Screener 是面向一位游戏玩家和最多 20 位已获授权好友的私密、低延迟屏幕分享工具。 +观看者只需浏览器，无须安装。媒体自动选择路径，优先 P2P，支持有容量限制的浏览器中继和 +内置 SFU/UDP 后备路径。Hosted Go 进程统一承载信令、STUN 和已授权的 SFU 媒体。 + +当前阶段以 Windows Client 和浏览器为验收目标。macOS、Linux 保留源码和构建支持， +实机验收另行进行。[当前状态](./docs/status.md) 区分源码能力、发布准备情况和最近记录的生产部署。 + +## 使用 Screener Client + +拿到相互匹配的 Client 程序包后： + +1. 完整解压，保留可执行文件旁的 `runtime` 目录。 +2. Windows 运行 `screener-client.exe`，Linux 运行 `./screener-client`， + macOS 打开 `Screener Client.app`。 +3. 选择模式和分享来源，将房间邀请链接发给好友。 + +Client 使用系统浏览器展示界面，没有内嵌浏览器界面。程序包包含 Go 应用以及原生采集、 +公网链接所需的辅助程序，运行时无须安装 Node.js、npm 或 Go。Linux 原生采集依赖系统的 +Portal、PipeWire 和 GStreamer，详见 [Client 指南](./cmd/screener-client/README.md)。 + +- **Local：** 与同一局域网中的设备分享。 +- **Public link：** 通过普通邀请链接分享。Cloudflare Quick Tunnel 只承载房间访问和信令， + 媒体仍通过 P2P 传输，不提供 SFU。链接只在本次 Client 运行期间有效，重启会创建新的临时链接， + Quick Tunnel 不保证持续可用。 +- **Site：** 打开已保存的 Screener 站点，并使用正在运行的 Client 进行原生采集或媒体转发。 + 站点可以提供 SFU 后备路径。 + +在来源选择器中选择浏览器、应用窗口或屏幕。原生屏幕音频来自默认系统输出；支持时， +窗口音频来自选中的应用。音频有独立开关。Windows 原生采集提供 VP8、Auto、H264 选项， +其他原生平台目前使用硬件 H.264。 + +原生节点复用适合直属观看者的编码输出，仅在需要时生成缺少的较低输出。每个节点的持续 +向外发送份数有上限，默认两份；不兼容的弱连接可以使用独立的本地编码输出。原生 SFU 发布 +复用同一来源。纯浏览器采集和中继仍使用正常的 WebRTC 发送器；编码复用属于原生节点能力。 +行为和边界见 [媒体质量](./docs/product/media-quality.md)。 + +媒体需要可用的 UDP 路径，P2P 不保证任意两个网络都能互通。浏览器或操作系统挂起、网络策略 +都可能中断分享。Client 本地站点默认开放，可在启动器中设置可选访问密码；这不会改变已保存 +Site 的访问规则。 + +Chromium 系浏览器无法建立媒体连接时，请查看 +[Chromium WebRTC 常见问题](./cmd/screener-client/README.md#chromium-webrtc-connections)。 +反馈问题时，以 `--debug` 启动 Client，复现后在终端按 `D` 导出本地诊断 ZIP。 +浏览器诊断单独导出，详见 [诊断与导出](./docs/reference/configuration.md#diagnostics)。 + +## 自行部署 + +服务器程序包运行 `screener-server`，浏览器资源已嵌入 Go 可执行文件。首次部署、配置及内置 +媒体服务的协同切换见 [自托管指南](./docs/operations/self-hosting.md)；匹配的构建产物、 +后续更新与恢复见 [部署与恢复](./docs/deployment.md)。文档中的流程不代表已经发布了公开的 +GitHub Release 或 Docker 镜像。 +[可选容器方案](./docs/operations/self-hosting.md#optional-container) 使用同一套已验证的服务器构建产物。 + +## 本地开发 + +需要 Node.js 24、npm 11 和 Go 1.26。 + +```sh +npm ci +npm run dev +``` + +在另一个终端启动 Go 服务。以下为 POSIX shell 写法，PowerShell 使用对应的 `$env:` 变量设置： + +```sh +PORT=8788 PUBLIC_BASE_URL=http://localhost:8787 go run ./cmd/screener-server +``` + +打开 `http://localhost:8787`。开发服务器监听 `0.0.0.0`，同一局域网设备可访问 +`http://<电脑局域网地址>:8787`；屏幕采集需在 `localhost` 或 HTTPS 上运行。 +需要不同邀请地址时，按 [配置参考](./docs/reference/configuration.md) 设置来源和访问规则。 +生产环境要求独立的 `SITE_ACCESS_PASSWORD`，实际凭据不要提交到仓库。 + +```sh +npm run check +npm run check:client +``` + +## 文档与许可 + +- [文档地图](./docs/README.md) +- [产品摘要](./docs/project-memory.md) +- [当前工作](./docs/todo.md) +- [贡献流程](./CONTRIBUTING.md) + +Screener 自有代码采用 [MIT 许可](./LICENSE)。分发的第三方组件保留各自许可和声明， +详见 [第三方许可](./licenses/README.md)。软件按“现状”提供，不作担保；性能和连通性取决于 +使用的设备、浏览器和网络。 diff --git a/cmd/screener-client/README.md b/cmd/screener-client/README.md index 87226fc..4654a8e 100644 --- a/cmd/screener-client/README.md +++ b/cmd/screener-client/README.md @@ -4,6 +4,20 @@ Screener Client is the self-contained and native-capability runtime for the same Browser application used by Hosted Screener. It does not implement another UI, room store, signaling protocol, or route controller. +## Run A Package + +Extract the matching platform bundle in full and keep `runtime` beside the +Client executable. Run `screener-client.exe` on Windows, `./screener-client` on +Linux, or `Screener Client.app` on macOS. The launcher opens in the system Browser; +the Client does not embed a browser UI. Packaged execution needs no Node.js, npm, +or Go installation. Linux native capture uses the system dependencies described +in the [Linux capture guide](../../native/capture/linux/README.md). + +Windows Client and Browser are this phase's acceptance targets. Other platform +builds do not establish physical capture/audio acceptance; see +[current status](../../docs/status.md). Package construction and public Release +publication are separate steps in [deployment](../../docs/deployment.md). + ## Modes - With no mode argument, the Client opens a small launcher in the system @@ -20,7 +34,7 @@ room store, signaling protocol, or route controller. opens and shows a notice when a newer full-SHA release exists. The check is best-effort and never installs or replaces the Client. -Local mode uses memory-only rooms, Browser P2P relay, no LiveKit, and no public +Local mode uses memory-only rooms, P2P relay, no SFU listener, and no public discovery. Ordinary Local works on a reachable LAN. The **Public invite** mode runs the packaged Cloudflare Tunnel sidecar for the existing HTTP/WebSocket control surface and @@ -34,8 +48,9 @@ TURN fallback. The Client chooses a sole private LAN IPv4 automatically. Use The system Browser remains the Host UI. A Client-launched Host offers the Browser's standard capture picker and a list of exact platform capture targets; the user selects one explicitly. Windows offers the same VP8/Auto/H264 selector: -VP8 uses libvpx, H264 uses hardware Media Foundation, and Auto measures target- -profile encoding work before choosing one codec for the share. Windows uses +VP8 uses the pinned WebRTC/libvpx encoder and H264 uses hardware Media Foundation +inside WebRTC's output pipeline. Auto measures target-profile encoding work +before choosing one codec for the share. Windows uses Graphics Capture and WASAPI; macOS uses ScreenCaptureKit, VideoToolbox, and AudioToolbox; Linux delegates selection to the ScreenCast Portal and uses the system PipeWire/GStreamer hardware path. @@ -47,21 +62,21 @@ connections. A Native edge with public STUN also attempts one bounded PCP, UPnP, or NAT-PMP mapping for its Pion UDP socket; pure LAN does not. Routers without a mapping service continue with ordinary ICE/STUN. The mapping does not create a relay or carry media through the Client control link. A configured Site -may route the native source through its existing Browser LiveKit publisher; +may receive a direct Native publication from the shared encoded source; Local and one-link modes remain P2P-only. An ordinary Web Host keeps the Browser capture path without probing the Client. -Native P2P edges normally reuse that one encoded source and negotiate transport- -wide feedback. Once Pion GCC has real feedback and the source has produced -frames, the Client reports whether that -edge's target payload bitrate can carry the measured shared video plus Opus -payload. The existing route controller owns persistence and any replacement; -the Client does not pace, score, or globally lower the shared encoder. If one -Native sender edge remains persistently degraded, the existing quality operation -may test a stock Browser WebRTC sender for that edge through the local bridge. -Existing Viewer evidence commits or rolls back the candidate; healthy Native -edges continue sharing the selected encode. Native Viewers can receive and -forward either H.264 or VP8 without encoding it again. +Native P2P edges and embedded SFU use one shared Pion/LiveKit media adapter for +feedback, forwarding allocation, bounded pacing and recovery. Native parents +reuse suitable H.264/VP8 outputs and derive a missing lower output only for +direct-child demand. Compatible children share that output; each edge receives +only its selected representation. SFU publication combines its requested output +prefix with one aggregate upstream budget. A lower-output constraint does not +replace the original input or higher sibling outputs. The existing route +controller still owns persistent quality evidence and any route replacement; +there is no room-wide score or periodic rebalance. See +[media quality](../../docs/product/media-quality.md) for the implemented behavior +and [status](../../docs/status.md) for its acceptance limits. The Client configuration keeps an optional Local site-access password. Leave it blank for an open Local site, or set a visible-ASCII password (8 to 128 bytes) @@ -70,21 +85,49 @@ Host page in a URL fragment; the page uses the existing SiteAccess endpoint and removes the fragment before continuing. Viewer invitations keep using the existing room-scoped grant. -The terminal starts in visual mode and shows the current mode, working entry -links and startup state. Its language follows the launcher and Client-enabled -pages; a block television uses the terminal foreground with golden sparkles, -falling back to ASCII without color support or in very narrow windows. -Visual mode depicts the selected mode and startup state with small scenes; -the television rests with one eye closed and occasionally winks while idle. +The terminal shows the current mode, entry links and startup state; its language +follows the launcher and Client-enabled pages. Press `o` to reopen the Browser, or `q` / Ctrl+C to end Local rooms and stop the local server and temporary public link. Plain-text output uses Ctrl+C. A Site-loaded Browser tab does not own the Client process. For one-link Internet sharing, open the Client launcher, choose **Public invite**, create a room in the opened Browser, and send its normal invitation link. The -random `trycloudflare.com` origin lasts only for that Client run. Cloudflare Quick -Tunnels provide no uptime guarantee; use a configured Site when persistent -control availability or SFU fallback matters. +random `trycloudflare.com` origin lasts only for that Client run; a later launch +creates a new temporary link. Cloudflare Quick Tunnels provide no uptime guarantee; +use a configured Site when persistent control availability or SFU fallback matters. + +## Troubleshooting + +### Chromium WebRTC Connections + +**Why can screen capture succeed while the media connection fails?** + +Chromium-based Browsers can restrict WebRTC UDP through Browser settings, +extensions or managed policies. Disabling non-proxied UDP can prevent even the +local Browser-to-Client media connection; successful capture or page loading +does not prove that this separate connection is available. This is not specific +to one Browser brand or to VPN use. + +Check the Browser's WebRTC/IP-handling policy and any extension's WebRTC or +IP-leak protection setting. Restore a policy that permits WebRTC UDP, reload +Screener, and verify that another extension or managed policy has not overridden +the choice. Setting names and availability differ between Browsers. For example, +Vivaldi exposes **Settings > Privacy and Security > WebRTC IP Handling > +Broadcast IP for Best WebRTC Performance**. Changing this policy can expose +network addresses to WebRTC peers; do not disable unrelated protections. + +See [Chromium's extension policy API](https://developer.chrome.com/docs/extensions/reference/api/privacy#property-network), +[Vivaldi's setting example](https://help.vivaldi.com/desktop/privacy/privacy-settings/), +and the [verified policy mechanism and field case](../../docs/research/native-client-lifecycle.md). + +### Diagnostics + +Start the packaged executable with `--debug`, reproduce the problem, then press +`D` in the interactive terminal to export a local ZIP. This does not stop the +share or upload the archive. Browser diagnostics are separate; the +[diagnostic reference](../../docs/reference/configuration.md#diagnostics) owns +log locations, export commands, retention and privacy boundaries. ## Development @@ -108,7 +151,11 @@ written to the ignored repository `build/client-check` directory and reused on the next run. This keeps the executable identity stable for the system firewall; the files are local build output and are never packaged or committed. -It runs Go formatting, unit tests, vet, and the three supported cross-builds. +It runs Go formatting, unit tests, vet, and Windows/Linux cgo-free cross-builds. +The Darwin Client and peer gate build only on macOS with cgo enabled and an +installed SDK; other hosts report that skipped platform explicitly. The pinned +media dependency's Darwin CPU statistics use Mach APIs through cgo, so a Windows +or Linux core check does not establish Darwin build acceptance. Each target compiles its isolated capture process and validates its bounded capability response. macOS additionally encodes one in-memory hardware H.264 IDR; Linux probes the Portal/PipeWire/GStreamer adapter. Real capture, GPU @@ -117,36 +164,38 @@ gates rather than environment-dependent unit tests. The loopback service binds IPv4 loopback on the first available port from `39721` through `39730`. `/health` discovers the current process; `/control` -accepts one strict v8 session. After `hello`, an available Client may list local -capture choices, own one generation-fenced Host share, or receive one native -Viewer source and its bounded encoded child edges. Its public `instanceToken` +admits at most two independent strict v9 control sessions. After `hello`, each +session may list local capture choices, own one generation-fenced Host share, +or receive one native Viewer source and its bounded encoded child edges. Closing +one session retires only its resources. Its public `instanceToken` distinguishes the discovered process but is not authentication; room authority and remote signaling remain in the Browser. Viewer receive/relay remains available even when this machine has no accepted native capture encoder. +Capture sidecars must match the Client's current probe/encoded-output contract; +the package-candidate wrapper validates that version before accepting its artifact. ## Packaging -Build one application release, then assemble a platform Client from that exact -descriptor. Native capture and the public-link sidecar are explicit package -inputs: +From a clean revision, build the application release and run the platform's +candidate wrapper on its native operating system. The wrapper builds capture, +verifies the pinned public-link helper, and assembles and checks the Client: ```sh node scripts/package-app-release.mjs /outside/repository/app-release -SCREENER_GO=/path/to/go \ - node scripts/assemble-client.mjs \ - /outside/repository/app-release/screener-.release.json \ - /outside/repository/Screener-Client \ - --target windows-amd64 \ - --capture /outside/repository/screener-client-capture.exe \ - --tunnel /outside/repository/cloudflared.exe +node scripts/package-client-candidate.mjs \ + /outside/repository/app-release windows-amd64 \ + /outside/repository/client-candidate ``` Supported targets are `windows-amd64`, `linux-amd64`, and `darwin-arm64`. -`--target` controls the Go cross-build and packaged executable names; the -capture and tunnel inputs must already match that target. Each target accepts -its matching native-capture input. +Use the matching target name in the command above. Darwin assembly requires a +native macOS runner with its SDK and enables cgo; Windows and Linux assembly keep +cgo disabled. +The result is a `tar.gz` bundle and SHA-256 file. Manual sidecar assembly, +explicit CI packaging, Release publication and updates are documented in +[deployment](../../docs/deployment.md); creating a candidate does not publish it. -The result contains: +The extracted bundle contains: ```text screener-client[.exe] @@ -243,7 +292,7 @@ probe and target OS support it. The cross-NAT variant uses a temporary reverse SSH path for signaling only and requires a selected `srflx` or `prflx` media pair; media never travels through SSH. The one-link media variant instead carries the same signaling through the Client's temporary public origin and requires direct -media delivery to an independent Linux peer. Native P2P quality evidence and the -Browser-mediated SFU path have dedicated gates. macOS and Linux capture still +media delivery to an independent Linux peer. Native P2P quality evidence and +embedded SFU delivery have explicit gates. macOS and Linux capture still require physical desktop/media gates; CI compilation and package smoke do not substitute for them. diff --git a/cmd/screener-client/main.go b/cmd/screener-client/main.go index f43f051..2d00bec 100644 --- a/cmd/screener-client/main.go +++ b/cmd/screener-client/main.go @@ -25,6 +25,8 @@ func main() { }) flag.BoolVar(&options.Local, "local", false, "use the self-contained Local room authority") flag.BoolVar(&options.Link, "link", false, "create one public Viewer invitation link") + flag.BoolVar(&options.Debug, "debug", false, "save opt-in Client diagnostics to rotated files") + flag.StringVar(&options.LogDir, "log-dir", "", "diagnostic directory (overrides SCREENER_LOG_DIR)") flag.StringVar(&options.CaptureProcess, "capture-process", "", "path to the platform native capture process") flag.StringVar(&options.TunnelProcess, "tunnel-process", "", "path to the packaged public tunnel process") flag.StringVar(&options.ConfigPath, "config", "", "path to the Client configuration file") diff --git a/cmd/screener-peer-gate/main.go b/cmd/screener-peer-gate/main.go index 998d8bf..5116dd6 100644 --- a/cmd/screener-peer-gate/main.go +++ b/cmd/screener-peer-gate/main.go @@ -13,16 +13,14 @@ import ( "sync" "time" + "github.com/TNTcraftHIM/Screener/internal/server/protocol" "github.com/coder/websocket" "github.com/coder/websocket/wsjson" "github.com/pion/interceptor" "github.com/pion/webrtc/v4" ) -const ( - signalingProtocol = "screener-v21" - maxPendingCandidates = 64 -) +const maxPendingCandidates = 64 type gateConfig struct { SignalURL string `json:"signalUrl"` @@ -168,7 +166,7 @@ func run(ctx context.Context, config gateConfig) gateResult { } defer receiver.close() if err = receiver.write(map[string]any{ - "type": "authenticate", "protocol": signalingProtocol, + "type": "authenticate", "protocol": protocol.SignalingProtocol, "roomId": config.RoomID, "role": "viewer", "clientId": config.ClientID, "viewerGrant": config.ViewerGrant, "viewerPresence": true, "displayName": "remote-gate", diff --git a/cmd/screener-server/diagnostics_unix.go b/cmd/screener-server/diagnostics_unix.go new file mode 100644 index 0000000..8548f31 --- /dev/null +++ b/cmd/screener-server/diagnostics_unix.go @@ -0,0 +1,33 @@ +//go:build unix + +package main + +import ( + "os" + "os/signal" + "syscall" + + "github.com/TNTcraftHIM/Screener/internal/diagnostics" +) + +func watchDiagnosticExport(recorder *diagnostics.Recorder) func() { + requests := make(chan os.Signal, 1) + stop, done := make(chan struct{}), make(chan struct{}) + signal.Notify(requests, syscall.SIGUSR1) + go func() { + defer close(done) + for { + select { + case <-stop: + return + case <-requests: + _ = exportServerDiagnostics(recorder) + } + } + }() + return func() { + signal.Stop(requests) + close(stop) + <-done + } +} diff --git a/cmd/screener-server/diagnostics_windows.go b/cmd/screener-server/diagnostics_windows.go new file mode 100644 index 0000000..c2687f3 --- /dev/null +++ b/cmd/screener-server/diagnostics_windows.go @@ -0,0 +1,6 @@ +package main + +import "github.com/TNTcraftHIM/Screener/internal/diagnostics" + +// Windows exports at orderly shutdown; it has no SIGUSR1 equivalent. +func watchDiagnosticExport(*diagnostics.Recorder) func() { return func() {} } diff --git a/cmd/screener-server/main.go b/cmd/screener-server/main.go index 0dd4fde..e98aa65 100644 --- a/cmd/screener-server/main.go +++ b/cmd/screener-server/main.go @@ -14,13 +14,16 @@ import ( "flag" "fmt" "io/fs" + "log" "log/slog" "os" "os/signal" + "path/filepath" "strings" "syscall" "time" + "github.com/TNTcraftHIM/Screener/internal/diagnostics" "github.com/TNTcraftHIM/Screener/internal/server/app" "github.com/TNTcraftHIM/Screener/internal/server/config" "github.com/TNTcraftHIM/Screener/internal/server/webassets" @@ -47,6 +50,7 @@ func main() { "path to the deployed REVISION file read by --check-release") apiURL := flag.String("api-url", defaultReleaseAPIURL, "release metadata endpoint used by --check-release") + debug := flag.Bool("debug", false, "save opt-in server diagnostics to rotated files") flag.Parse() if err := loadEnvironmentFile(environmentFile); err != nil { @@ -61,7 +65,7 @@ func main() { } fmt.Println("config=ok") default: - if err := serve(); err != nil { + if err := serve(*debug); err != nil { fail(err) } } @@ -75,7 +79,31 @@ func fail(err error) { // serve runs the application until the first SIGINT or SIGTERM, then stops it // with Close rather than End: Hosted rooms are leased in the database and must // survive a restart (DECISIONS D6). -func serve() error { +func serve(debug bool) (returnedErr error) { + logger := slog.New(slog.NewTextHandler(os.Stderr, nil)) + if debug || serverDebugEnabled(os.Getenv("SCREENER_DEBUG")) { + recorder, err := diagnostics.Open(serverLogDirectory(), "server", BuildRevision) + if err != nil { + return fmt.Errorf("Screener server diagnostics are unavailable: %w", err) + } + logger = slog.New(slog.NewMultiHandler(logger.Handler(), recorder.Logger().Handler())) + previous, previousWriter, previousFlags := slog.Default(), log.Writer(), log.Flags() + slog.SetDefault(logger) + // Unstructured dependency logs retain their existing journal destination. + log.SetOutput(previousWriter) + log.SetFlags(previousFlags) + fmt.Fprintln(os.Stderr, "Screener diagnostic log:", recorder.LogPath()) + stopExport := watchDiagnosticExport(recorder) + defer func() { + stopExport() + logger.Info("screener-server", "event", "stopped", "failed", returnedErr != nil) + returnedErr = errors.Join(returnedErr, exportServerDiagnostics(recorder), recorder.Close()) + slog.SetDefault(previous) + log.SetOutput(previousWriter) + log.SetFlags(previousFlags) + }() + logger.Info("screener-server", "event", "start", "revision", BuildRevision) + } configuration, err := config.Load(environment()) if err != nil { return err @@ -83,7 +111,7 @@ func serve() error { server, err := app.New(app.Options{ Config: configuration, Assets: webassets.FS(), - Logger: slog.New(slog.NewTextHandler(os.Stderr, nil)), + Logger: logger, }) if err != nil { return err @@ -109,6 +137,36 @@ func serve() error { return server.Close(shutdownCtx) } +func serverDebugEnabled(value string) bool { + for _, component := range strings.Split(value, ",") { + switch strings.TrimSpace(component) { + case "server", "route": + return true + } + } + return false +} + +func serverLogDirectory() string { + if directory := strings.TrimSpace(os.Getenv("SCREENER_LOG_DIR")); directory != "" { + return directory + } + if directories := filepath.SplitList(os.Getenv("LOGS_DIRECTORY")); len(directories) > 0 && directories[0] != "" { + return directories[0] + } + return "logs" +} + +func exportServerDiagnostics(recorder *diagnostics.Recorder) error { + path, err := recorder.Export() + if err != nil { + fmt.Fprintln(os.Stderr, "Screener diagnostic export failed:", err) + return err + } + fmt.Fprintln(os.Stderr, "Screener diagnostic bundle:", path) + return nil +} + // environment is the process environment in the form config.Load expects. func environment() map[string]string { values := make(map[string]string) diff --git a/cmd/screener-server/main_test.go b/cmd/screener-server/main_test.go index 9bef26d..94f8e58 100644 --- a/cmd/screener-server/main_test.go +++ b/cmd/screener-server/main_test.go @@ -65,3 +65,27 @@ func TestLoadEnvironmentFileIgnoresOnlyAMissingFile(t *testing.T) { t.Fatal("an unreadable environment file was ignored") } } + +func TestServerDiagnosticSelection(t *testing.T) { + for value, want := range map[string]bool{ + "server": true, "route": true, "client, route": true, + "": false, "client": false, "all": false, "server-secret": false, + } { + if got := serverDebugEnabled(value); got != want { + t.Fatalf("serverDebugEnabled(%q) = %v, want %v", value, got, want) + } + } + t.Setenv("SCREENER_LOG_DIR", "") + t.Setenv("LOGS_DIRECTORY", "") + if got := serverLogDirectory(); got != "logs" { + t.Fatalf("default directory = %q", got) + } + t.Setenv("LOGS_DIRECTORY", "service-logs"+string(os.PathListSeparator)+"other-logs") + if got := serverLogDirectory(); got != "service-logs" { + t.Fatalf("systemd directory = %q", got) + } + t.Setenv("SCREENER_LOG_DIR", "explicit-logs") + if got := serverLogDirectory(); got != "explicit-logs" { + t.Fatalf("explicit directory = %q", got) + } +} diff --git a/deploy/container/Dockerfile b/deploy/container/Dockerfile new file mode 100644 index 0000000..b794cc7 --- /dev/null +++ b/deploy/container/Dockerfile @@ -0,0 +1,8 @@ +FROM gcr.io/distroless/static-debian13:nonroot-amd64@sha256:e754765ad9e167b0677b41c617fd44afb7b9818a477f48f17bda08e12cfb98cb +WORKDIR /app +COPY --chmod=0555 screener-server /app/ +COPY --chmod=0444 LICENSE THIRD-PARTY-NOTICES.txt REVISION /app/ +ENV LISTEN_HOST=0.0.0.0 SCREENER_LOG_DIR=/home/nonroot/logs +USER 65532:65532 +EXPOSE 8787/tcp 3478/udp 3479/udp 3480/udp 7882/udp +ENTRYPOINT ["/app/screener-server"] diff --git a/deploy/coturn/turnserver.conf.example b/deploy/coturn/turnserver.conf.example deleted file mode 100644 index b75062c..0000000 --- a/deploy/coturn/turnserver.conf.example +++ /dev/null @@ -1,33 +0,0 @@ -# Screener self-hosted STUN-only configuration. -# Target: coturn 4.17.2 or a newer patched release. -# Copy this file outside the repository and restrict it to the coturn service -# account. No application secret or relay port range is required. - -# Ordinary peer ICE uses UDP STUN discovery only. Coturn documents `stun-only` -# as ignoring every TURN request, so this listener cannot allocate media relays. -listening-port=3478 -stun-only -no-tcp -no-tls - -# Approved NAT survey listeners only. Keep `stun-only` enabled: these auxiliary -# endpoints are for ordinary Binding requests, not TURN media relay. Replace -# with the host address already bound by the main listener. -# They use the same public IP and therefore measure destination-port behavior; -# full RFC 5780 behavior discovery still needs a second public IPv4. -#aux-server=:3479 -#aux-server=:3480 - -# Keep automatic address selection on a single-homed public host. On a -# multi-homed host, bind only the intended public interface. -#listening-ip= - -fingerprint -no-cli - -# Let the service supervisor apply bounded retention. Do not enable verbose or -# log-binding in normal production; STUN logs can contain client IP addresses. -log-file=stdout -log-min-level=info -new-log-timestamp -pidfile=/run/coturn/turnserver.pid diff --git a/deploy/livekit/livekit.yaml.example b/deploy/livekit/livekit.yaml.example deleted file mode 100644 index ee663f9..0000000 --- a/deploy/livekit/livekit.yaml.example +++ /dev/null @@ -1,43 +0,0 @@ -# Minimal single-node LiveKit 1.13.6 capacity for Screener's optional automatic -# SFU fallback. Copy this file to an untracked service-owned location such as -# /etc/livekit/livekit.yaml. Store only the generated key mapping in -# /etc/livekit/keys.yaml, owned by the livekit service user with mode 0600. - -port: 7880 -key_file: /etc/livekit/keys.yaml - -rtc: - # These media ports must be reached directly by clients. Port 7880 is only - # for the same-host reverse proxy and must be denied by every public firewall. - udp_port: 7882 - tcp_port: 0 - allow_tcp_fallback: false - # Use the deployment-owned STUN listener for server public-IP discovery and - # avoid LiveKit substituting a public default in its join response. The - # Screener Browser SFU connects use an empty ICE-server list; ordinary peer - # edges receive this listener through application signaling. - stun_servers: - - stun.example.com:3478 - use_external_ip: true - congestion_control: - # Let LiveKit select the forwarded layer from subscriber bandwidth. - use_send_side_bwe: true - -room: - # Screener creates exact-generation rooms through the private RoomService API. - # Keep this false so a stale participant token cannot recreate a drained room. - auto_create: false - # Screener enforces its configured admission limit before issuing a token. - # This server-side ceiling covers the repository's 20 viewers plus one Host. - max_participants: 21 - -logging: - # Info-level join records include raw PublisherOffer SDP in LiveKit 1.13.5. - # Keep production at warn so credentials and media-path details stay out of - # the persistent journal; raise it only in an isolated, short-lived canary. - level: warn - pion_level: error - -# External or embedded TURN, Redis, recording, ingress, egress, and webhooks are -# omitted on purpose. ICE/TCP fallback is explicitly disabled above. This -# candidate is one process on one node. diff --git a/deploy/nginx/share.bonfire.icu.conf.example b/deploy/nginx/share.bonfire.icu.conf.example index 188a76a..c5906b9 100644 --- a/deploy/nginx/share.bonfire.icu.conf.example +++ b/deploy/nginx/share.bonfire.icu.conf.example @@ -39,46 +39,9 @@ server { add_header X-Frame-Options "DENY" always; add_header Permissions-Policy "camera=(), microphone=(), geolocation=()" always; - # Reject the unversioned fallback endpoint that pinned livekit-client@2.22.0 - # can still try. Keep it out of logs because the URL may contain a join token. - location = /rtc { - access_log off; - error_log /dev/null crit; - return 404; - } - - # Pinned livekit-client@2.22.0 still tries this v0 endpoint after a - # ServiceNotFound response. Reject that fallback with a no-log 404; - # /rtc/v1* below is the accepted SDK path. - location = /rtc/validate { - access_log off; - error_log /dev/null crit; - return 404; - } - - # This integration accepts only the v1 path. Set LIVEKIT_URL to - # wss://share.bonfire.icu. - location ^~ /rtc/v1 { - # The WebSocket query contains a short-lived LiveKit join token. Do not - # persist the request target in either access or error logs. - access_log off; - error_log /dev/null crit; - proxy_pass http://127.0.0.1:7880; - proxy_http_version 1.1; - proxy_set_header Host $host; - proxy_set_header X-Forwarded-Host $host; - proxy_set_header X-Forwarded-Proto https; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - proxy_set_header Upgrade $http_upgrade; - proxy_set_header Connection $screener_connection_upgrade; - proxy_read_timeout 5h; - proxy_send_timeout 5h; - proxy_buffering off; - } - - # Reject any other retired or unknown LiveKit path without logging tokens. - location ^~ /rtc/ { + # Media uses /signal. Reject credential-bearing RTC query endpoints without + # persisting their query strings, including requests from already-open pages. + location ~ ^/rtc(?:/|$) { access_log off; error_log /dev/null crit; return 404; diff --git a/deploy/release-app.sh b/deploy/release-app.sh index 8a6dd5f..37e8906 100644 --- a/deploy/release-app.sh +++ b/deploy/release-app.sh @@ -150,10 +150,15 @@ test -f "$artifact" test -f "$manifest" test "$(sha256sum "$artifact" | awk '{print $1}')" = "$artifact_sha" test "$(sha256sum "$manifest" | awk '{print $1}')" = "$manifest_sha" -for service in screener livekit coturn nginx; do +for service in screener nginx; do test "$(systemctl show "${service}.service" -p ActiveState --value)" = 'active' test "$(systemctl show "${service}.service" -p NRestarts --value)" = '0' done +# The first Go/embedded-media cutover owns its infrastructure recovery separately. +test -x "$old_release/screener-server" +old_pid="$(systemctl show screener.service -p MainPID --value)" +test "$old_pid" -gt 1 +test "$(readlink -f "/proc/${old_pid}/exe")" = "$old_release/screener-server" while IFS= read -r raw_entry; do entry="$raw_entry" @@ -228,8 +233,6 @@ test ! -L "$release" test "$(stat -c '%U:%G %a' "$release")" = 'root:root 755' firewall_before="$(nft list table inet bonfire_filter | sha256sum | awk '{print $1}')" -livekit_restarts="$(systemctl show livekit.service -p NRestarts --value)" -coturn_restarts="$(systemctl show coturn.service -p NRestarts --value)" nginx_restarts="$(systemctl show nginx.service -p NRestarts --value)" cutover_since="$(date '+%Y-%m-%d %H:%M:%S')" cutover_start="$(date +%s%3N)" @@ -245,8 +248,6 @@ health_ready="$(date +%s%3N)" test "$(systemctl show screener.service -p ActiveState --value)" = 'active' test "$(systemctl show screener.service -p NRestarts --value)" = '0' -test "$(systemctl show livekit.service -p NRestarts --value)" = "$livekit_restarts" -test "$(systemctl show coturn.service -p NRestarts --value)" = "$coturn_restarts" test "$(systemctl show nginx.service -p NRestarts --value)" = "$nginx_restarts" test "$(nft list table inet bonfire_filter | sha256sum | awk '{print $1}')" = "$firewall_before" test "$(readlink -f -- "$current")" = "$release" diff --git a/deploy/systemd/coturn.service.example b/deploy/systemd/coturn.service.example deleted file mode 100644 index 9c4b945..0000000 --- a/deploy/systemd/coturn.service.example +++ /dev/null @@ -1,39 +0,0 @@ -[Unit] -Description=Screener STUN discovery service -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=turnserver -Group=turnserver -RuntimeDirectory=coturn -RuntimeDirectoryMode=0750 -ExecStartPre=/usr/bin/test -r /etc/screener/turnserver.conf -ExecStart=/usr/local/bin/turnserver -c /etc/screener/turnserver.conf -Restart=on-failure -RestartSec=3s -LimitNOFILE=65536 -UMask=0077 - -NoNewPrivileges=true -PrivateDevices=true -PrivateTmp=true -ProtectSystem=strict -ProtectHome=true -ProtectKernelTunables=true -ProtectKernelModules=true -ProtectKernelLogs=true -ProtectControlGroups=true -ProtectClock=true -ProtectHostname=true -RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK -RestrictNamespaces=true -RestrictRealtime=true -RestrictSUIDSGID=true -LockPersonality=true -CapabilityBoundingSet= -SystemCallArchitectures=native - -[Install] -WantedBy=multi-user.target diff --git a/deploy/systemd/livekit.service.example b/deploy/systemd/livekit.service.example deleted file mode 100644 index 825ca4a..0000000 --- a/deploy/systemd/livekit.service.example +++ /dev/null @@ -1,39 +0,0 @@ -[Unit] -Description=LiveKit single-node SFU fallback for Screener -After=network-online.target -Wants=network-online.target - -[Service] -Type=simple -User=livekit -Group=livekit -ExecStartPre=/usr/bin/test -r /etc/livekit/livekit.yaml -ExecStartPre=/usr/bin/test -r /etc/livekit/keys.yaml -ExecStart=/usr/local/bin/livekit-server --config /etc/livekit/livekit.yaml -Restart=on-failure -RestartSec=3s -TimeoutStopSec=20s -UMask=0077 -LimitNOFILE=65535 - -NoNewPrivileges=true -PrivateDevices=true -PrivateTmp=true -ProtectSystem=strict -ProtectHome=true -ProtectKernelTunables=true -ProtectKernelModules=true -ProtectKernelLogs=true -ProtectControlGroups=true -ProtectClock=true -ProtectHostname=true -RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX AF_NETLINK -RestrictNamespaces=true -RestrictRealtime=true -RestrictSUIDSGID=true -LockPersonality=true -CapabilityBoundingSet= -SystemCallArchitectures=native - -[Install] -WantedBy=multi-user.target diff --git a/deploy/systemd/screener-livekit-readiness.conf.example b/deploy/systemd/screener-livekit-readiness.conf.example deleted file mode 100644 index 1fe3573..0000000 --- a/deploy/systemd/screener-livekit-readiness.conf.example +++ /dev/null @@ -1,9 +0,0 @@ -# Install as /etc/systemd/system/screener.service.d/livekit-readiness.conf only -# when this Screener instance is configured with the same-host LiveKit owner. - -[Unit] -Wants=livekit.service -After=livekit.service - -[Service] -ExecStartPre=/bin/sh -c 'attempt=0; until curl -sS --max-time 1 -o /dev/null http://127.0.0.1:7880 2>/dev/null; do attempt=$((attempt + 1)); [ "$attempt" -ge 30 ] && exit 1; sleep 1; done' diff --git a/deploy/systemd/screener.service.example b/deploy/systemd/screener.service.example index 4c408d8..0e62bca 100644 --- a/deploy/systemd/screener.service.example +++ b/deploy/systemd/screener.service.example @@ -1,6 +1,5 @@ [Unit] -Description=Screener WebRTC signaling and web application -# Same-host LiveKit deployments also install the tracked readiness drop-in. +Description=Screener web, signaling, STUN and SFU application After=network-online.target Wants=network-online.target @@ -18,6 +17,8 @@ TimeoutStopSec=20s UMask=0077 StateDirectory=screener StateDirectoryMode=0700 +LogsDirectory=screener +LogsDirectoryMode=0700 NoNewPrivileges=true PrivateDevices=true diff --git a/docs/adr/0005-automatic-hybrid-media-routing.md b/docs/adr/0005-automatic-hybrid-media-routing.md index 2d2a354..64cbe90 100644 --- a/docs/adr/0005-automatic-hybrid-media-routing.md +++ b/docs/adr/0005-automatic-hybrid-media-routing.md @@ -4,6 +4,12 @@ - Date: 2026-08-20 - Last updated: 2026-08-30 +The current candidate replaces this ADR's external LiveKit room/token lifecycle +with admitted in-process SFU resources under [ADR-0013](./0013-embedded-node-local-media.md). +Exact physical subscription closure now releases its reservation. The graph, +capacity and serial-operation decisions below remain; external-service details +describe the earlier implementation and its production history. + ## Context Direct Browser WebRTC gives Screener its desired latency and distributed cost, diff --git a/docs/adr/0007-path-isolated-representation-quality.md b/docs/adr/0007-path-isolated-representation-quality.md index 54a11f7..ffecd29 100644 --- a/docs/adr/0007-path-isolated-representation-quality.md +++ b/docs/adr/0007-path-isolated-representation-quality.md @@ -4,6 +4,12 @@ - Date: 2026-08-19 - Last reviewed: 2026-09-01 +[ADR-0013](./0013-embedded-node-local-media.md) supersedes the external SFU SDK, +Dynacast-service and single-output Native assumptions for the current candidate. +Browser source intent, codec selection and framework-owned adaptation remain. +Historical evidence below is not acceptance of the shared Native/embedded-SFU +implementation; [media quality](../product/media-quality.md) owns current behavior. + ## Context Screener sends one realtime game-screen source through independent P2P diff --git a/docs/adr/0010-cross-platform-client-runtime.md b/docs/adr/0010-cross-platform-client-runtime.md index b194ff0..320bc5c 100644 --- a/docs/adr/0010-cross-platform-client-runtime.md +++ b/docs/adr/0010-cross-platform-client-runtime.md @@ -5,7 +5,13 @@ - Superseded in part: items 1, 5, 7, 12 (its supervisor sentence), 13, 14, and the local-package consequence were amended in place by [ADR-0012](./0012-shared-go-backend-core.md), which replaced the TypeScript - server with one shared Go core. Every other item stands as written. + server with one shared Go core. +- [ADR-0013](./0013-embedded-node-local-media.md) supersedes item 8's single-output, + Pion-GCC-observation-only and Browser-mediated LiveKit publisher details for + the current candidate. The shared encoded-media adapter and direct Native SFU + publisher preserve this ADR's participant and capability boundaries. The + independent bounded control-session contract remains current; historical + acceptance below does not validate the new media implementation. ## Context @@ -33,14 +39,18 @@ topologies and makes a saved Site unavailable while another room source runs. [ADR-0012](./0012-shared-go-backend-core.md). 2. The packaged product is Screener Client. Its Go process is the cross-platform entry and native capability provider. It starts `/health` and one `/control` - WebSocket on IPv4 loopback within ports `39721` through `39730` before room- + endpoint on IPv4 loopback within ports `39721` through `39730` before room- source selection and retains them until the process exits. 3. Discovery returns a per-process `instanceToken`. The WebSocket subprotocol echoes it so the Browser connects to the process it discovered. This value is public process identity, not authentication. The listener accepts the current Local Host origin and the one user-saved Site origin. Origin and Host validation plus the Browser's local-network permission own this boundary. -4. Loopback v8 starts with a strict `hello` handshake. Health discovery reports +4. Loopback v9 starts with a strict `hello` handshake. Up to two participant + control sessions may coexist; each retains its own share and media lifetime. + This bounds concurrent Native work, not the shared server's room count. + Ending one control session cannot end another; Client exit waits for all + owned sessions to retire. Health discovery reports only separately probed native capture booleans. Native Viewer receive/NAT remains available when capture or hardware encode is absent. An active control session may list local screen/window choices, request bounded previews, and own one share's generation-fenced SDP/ICE @@ -65,9 +75,9 @@ topologies and makes a saved Site unavailable while another room source runs. Client again. 7. A self-contained Local deployment serves reachable LAN peers without a central Screener service. Its explicit `--link` mode starts one accountless - Cloudflare Quick Tunnel for the same HTTP/WebSocket surface, applies the - resulting HTTPS origin to the local server configuration before that server - starts listening, and otherwise retains the same + Cloudflare Quick Tunnel for the same HTTP/WebSocket surface. The Client + reserves the exact IPv4 listener before creating that tunnel, then applies + its HTTPS origin before the server starts serving that listener. It retains the same memory RoomStore, Browser UI, Viewer grant, signaling, and route controller. The Host sends the ordinary invitation link and the Viewer needs only a Browser. Cloudflare terminates this temporary control path; WebRTC media stays diff --git a/docs/adr/0011-browser-assisted-native-fanout.md b/docs/adr/0011-browser-assisted-native-fanout.md index d7dcb67..b5b63e2 100644 --- a/docs/adr/0011-browser-assisted-native-fanout.md +++ b/docs/adr/0011-browser-assisted-native-fanout.md @@ -6,13 +6,17 @@ ## Context The product media goal is one encoded source per capable distribution layer. -Pure Browser WebRTC does not expose a way to inject a received encoded frame +Standard Browser WebRTC does not expose a way to inject a received encoded frame into another `RTCPeerConnection` sender. `RTCRtpScriptTransform` can observe or modify an existing sender or receiver pipeline, but it does not provide an RTP transport fanout API. Browser simulcast and Dynacast remain appropriate for a single Browser-to-SFU publication; they do not make independent P2P senders share one encoder. +The later [Chromium legacy probe](../research/advanced-peer-distribution.md#chromium-legacy-fanout-probe) +demonstrates a non-standard clean-path exception, with unresolved feedback and +statistics. It is not an accepted replacement for this Browser/Client boundary. + The repository already has the required Native boundary: a Pion receiver can accept H.264/Opus RTP and its encoded source can feed bounded downstream Pion edges without decoding or re-encoding. The system Browser remains the product diff --git a/docs/adr/0013-embedded-node-local-media.md b/docs/adr/0013-embedded-node-local-media.md new file mode 100644 index 0000000..98f332d --- /dev/null +++ b/docs/adr/0013-embedded-node-local-media.md @@ -0,0 +1,122 @@ +# ADR-0013: Embedded Node-Local Media + +- Status: Accepted; physical evidence and release boundaries are owned by status +- Date: 2026-09-07 + +## Context + +The owner accepts one self-contained Screener process for the current STUN and +SFU services, and one shared encoded-media model for capable Native P2P nodes +and the SFU. Bundling or supervising coturn/LiveKit executables does not meet +this requirement. The existing product uses coturn only as STUN, not TURN. + +The [encoding comparison](../research/node-local-encoding-probe.md) and +[decode comparison](../research/node-local-browser-probe.md) establish real +unused-layer cost and codec lifecycle costs, but not complete network/hardware +acceptance. Mature simulcast implementations provide the starting model. + +## Decision + +1. Every parent, including Host and the SFU, owns only its direct children's + media demands. Keep the existing single room authority, committed route graph + and serial route operation. Media layer changes are not topology operations. +2. Forward a suitable existing encoding. Derive a missing lower output at its + direct parent and share that encoded output among compatible children. Do + not create an encoder per child. A forwarding-only healthy relay creates no + decoder/encoder group just to manufacture unused fallback layers. +3. An already-needed encoding group uses a finite highest-demand envelope: + produce the highest required output and its lower fallbacks, and stop upper + outputs no child needs. This is not a promise of a single Host encode under + all conditions. Representation settings come from the chosen media component + within Host ceilings, not a new Screener quality score or arbitrary ladder. +4. Each child receives only its selected output. Multiple active encodings do + not imply sending every output to every child. Preserve the original input, + source clock and unaffected siblings. Playback quality, cadence and recovery + take priority over reducing the number of encoders. +5. Mature media components own bandwidth estimation, rate allocation, pacing, + loss recovery and safe switching. The shared group owns representation + lifetime and reuse. Do not add a second congestion controller, fine-grained + warm cache, whole-tree demand scanner or periodic topology rebalancer. +6. The SFU remains one Host publication with one-to-many distribution. For a + forwarding SFU, its upstream demand is the maximum required by its direct + subscribers; Host combines that demand with its direct P2P children's demand. + This requests the highest needed representation, not a sum of encoding counts + or a room-wide minimum. The aggregate publication must fit its own upstream + congestion budget; requested quality is not a delivery guarantee. +7. Shared group code must be reusable by the SFU and Native paths. A future + transcoding SFU reports its actual input requirements instead of blindly + forwarding every downstream request. Do not require server transcoding or + recursive subtree demand to ship a forwarding SFU. +8. Embed library-level STUN, preserving Binding and the three NAT-survey + destinations without enabling TURN allocations. Evaluate an embedded SFU + library through a narrow adapter. `inlivedev/sfu` is a candidate, not an + irrevocable selection or authorization for a deep fork. +9. Pure Browser peers retain normal WebRTC capture/senders and can consume the + same H264/VP8 output. No Browser encoded-passthrough promise, new codec family + or application transport is introduced by this architecture. + +For an SFU that forwards existing layers: + +```text +SFU upstream demand = max(SFU direct-subscriber demands) +Host output demand = max(Host direct-P2P demands, SFU upstream demand) +``` + +Indirect P2P descendants are not included in either aggregate. Native parents +handle their own derivation. Framework estimates must distinguish desired, +available and actually delivered output so missing high input cannot be +mistaken for a healthy high-quality path. + +## Consequences And Release Boundary + +The Native integration uses one complete stock WebRTC encoding/adaptation +pipeline per shared local output. Its source adapter, rate correction, resource +feedback and input dropping share that output's lifetime; they are not cloned +per consumer behind an encoder cache. The existing Pion/LiveKit transport owns +the input bitrate budget and forwarding. Ordinary encoder adaptation preserves +the unadapted capture/received original, source timeline and route identity. +This refines the implementation boundary, not the direct-child demand model or +an acceptance claim. The [encoder-pool evidence](../research/webrtc-encoder-pool.md) +shows why sharing the codec alone loses reuse or churns under independent drops. + +Incompatible direct-child budgets own separate adaptive outputs. Equal effective +requests within the same source generation, codec, preference and ceiling may +share; dimensions alone are not a compatibility key. Rates update an owned +pipeline in place, not by recreating it for every bitrate sample. Membership +changes wait for independent recovery, retain existing transport identity and +stop the last unused codec slot. The original input is not lowered by a derived +group. Physical codec slots are bounded by endpoint admission, not by LiveKit's +three spatial-layer labels; each connection sees original plus its assigned +adaptive output. SFU publication retains its aggregate allocation separately. +No ancestor lookup, extra upstream layer subscription or new congestion policy +is introduced. + +- One media model can serve Native edges and embedded server forwarding without + duplicating room authority. The adapter is not a second room policy. +- Extra low encodes, decoder work, frame copies and publication upload remain + real costs. A fixed set of layers does not solve bandwidth below its lowest + layer or CPU overload; those cases need framework adaptation and verification. +- An available lowest output owned by a live local rate-controlled codec uses + the framework's non-pausing allocation policy at positive budgets; raw relay + input and forwarding SFU outputs retain pausing. Cold codec preparation and + existing estimation windows still impose transition cost, so eventual recovery + does not establish low-bandwidth latency parity. +- Replacing the external SFU changes Browser signaling/media integration and + deployment. Cut over one coordinated current contract, delete replaced paths + in that boundary, and preserve active-share/rollback rules from CONTRIBUTING. +- The candidate implements this model through shared Pion/LiveKit media + components; ADR-0007 retains the earlier + production/runtime context and its unchanged Browser source/codec decisions. + Implementation is not acceptance of hardware, mixed peers, loss/recovery or + production cutover. Current product modules describe the candidate, while + status and deployment records keep production evidence separate. +- [TODO](../todo.md) owns remaining implementation and acceptance. Research owns + measurements and library-selection evidence; no production switch is added + merely because a probe passes. + +## References + +- [RTP simulcast design and tradeoffs, RFC 8853](https://www.rfc-editor.org/rfc/rfc8853) +- [RTP switching and source projection, RFC 7667](https://www.rfc-editor.org/rfc/rfc7667) +- [LiveKit demand envelope](https://github.com/livekit/livekit/blob/v1.13.6/pkg/rtc/dynacast/dynacastmanagervideo.go) +- [Implementation proposal and reuse map](../research/node-local-media-design.md) diff --git a/docs/deployment.md b/docs/deployment.md index 97fba48..192439f 100644 --- a/docs/deployment.md +++ b/docs/deployment.md @@ -7,17 +7,17 @@ identity is owned by the immutable release descriptor, runtime `REVISION`, and deployment record. [Status](./status.md) owns only the compact current product and operational snapshot; it is not a per-release ledger. -The tracked `release-app.sh` is the updater for the current bare-metal nginx, -LiveKit, coturn, nftables, and local-port-8787 deployment. It requires an -existing current release. Another proxy, optional-LiveKit shape, firewall owner, -application port, or first installation needs its own scoped bootstrap/updater; -do not call this wrapper generic. +The tracked `release-app.sh` updates an existing Go deployment with embedded +media, bare-metal nginx, nftables, and application port 8787. It requires an +existing running Go release. Another proxy, firewall owner, application port, +first installation, or initial embedded-media cutover needs its own scoped +bootstrap/updater; do not call this wrapper generic. ## Release Boundary A routine application release changes only the built Screener server binary and the Browser assets it embeds. It does not change infrastructure, -service units, proxy/firewall rules, secrets, LiveKit/coturn configuration, or +service units, proxy/firewall rules, secrets, media listener configuration, or persistent room state. Any task that touches one of those excluded surfaces must first record its exact @@ -64,6 +64,13 @@ is one of `windows-amd64`, `linux-amd64`, or `darwin-arm64`; every supplied binary input must match it. It does not create an installer, auto-updater, release tag, or compatibility bundle. +Darwin Client assembly runs on macOS with its SDK and cgo enabled: the pinned +LiveKit dependency's [Darwin CPU statistics](https://github.com/mackerelio/go-osstat/blob/v0.2.8/cpu/cpu_darwin_cgo.go) +call Mach APIs. Windows and Linux +Client builds keep cgo disabled. A core check on those hosts explicitly skips +the Darwin binaries; macOS build and package verification belong to the existing +native runner. This does not change the cgo-free linux/amd64 Server artifact. + The platform package also carries its native presentation metadata: Windows embeds the icon in the Go executable, Linux emits a freedesktop desktop entry under `share/`, and macOS emits a thin `.app` launcher with an ICNS resource. @@ -88,11 +95,11 @@ Retain the descriptor and successful deployment output as release metadata. Do not create a follow-up source commit solely to duplicate their revision, asset, or hashes. -After `validate` succeeds on a push to `main`, CI packages one short-lived -Server application release. The three-platform Client matrix is an explicit -manual workflow dispatch with `client_checks=true`; it consumes that same -application artifact and retains the candidates for 14 days. Neither path is a -tag, public GitHub Release, or deployment. +After `validate` succeeds on an explicit workflow dispatch with +`client_checks=true`, CI packages the Server application release and the +three-platform Client candidates that consume it. The artifacts are retained +for 14 days. Ordinary `main` pushes do not run these packaging jobs. This +workflow does not create a tag, public GitHub Release, or deployment. Do not build or run the full repository check on a constrained production host. That host runs only the packaged binary and needs no Node, npm, or dependency @@ -130,13 +137,16 @@ state. A different current-revision file may be supplied as its only argument. ## Atomic Cutover -### First Go cutover prerequisite +### First embedded-media cutover prerequisite -The release wrapper is an application updater, not a systemd installer. Before -the first Go release, install and reload the tracked unit as described in the -[self-hosting cutover procedure](./operations/self-hosting.md#first-go-service-cutover). -Do not mix the new Go binary with the old Node unit or an environment file that -still sets `NODE_ENV`. +The first move from external services to embedded STUN/SFU is a coordinated +infrastructure and protocol transaction, including Node-to-Go where still +needed. Complete the [self-hosting cutover procedure](./operations/self-hosting.md#coordinated-embedded-media-cutover) +with matching `screener-v22` Web/Server and native protocol v9 Client builds, +accepted active-session interruption, released UDP ports, and exact +unit/environment/proxy/service recovery. The routine wrapper does not perform +that transaction; it requires an already running packaged Go release and cannot +restore the previous infrastructure. Run the tracked server entry with the uploaded descriptor: @@ -149,7 +159,7 @@ sudo env SCREENER_PUBLIC_ORIGIN=https://share.example.com \ The wrapper: 1. takes a deployment lock and validates the descriptor, revision, archive, - manifest, current release, required services, and public origin; + manifest, current Go process, Screener/nginx service state, and public origin; 2. rejects unexpected archive paths, links, file types, or duplicate inodes; 3. extracts to a new release directory and verifies every file hash and size; 4. validates production configuration by running the packaged binary's @@ -174,7 +184,7 @@ process state. Lightweight mode has no persistent room state and starts empty. A release does not add a migration or compatibility reader for either mode; current exact -schema validation fails closed before signaling or LiveKit mutation. +schema validation fails closed before accepting signaling or media admission. ## Postflight @@ -182,8 +192,9 @@ Verify only the surfaces relevant to the release: - local and public `/healthz` return `{"status":"ok"}`; - the public HTML references the new immutable main asset; -- Screener, nginx, coturn, and LiveKit have expected active/restart - state; +- Screener and nginx have expected active/restart state; +- Screener owns each configured STUN/SFU UDP listener and public reachability + matches the configuration; - the current symlink and served revision match the descriptor; - room creation, Host authentication, one direct Viewer, and configured SFU fallback complete the scoped smoke; and diff --git a/docs/operations/self-hosting.md b/docs/operations/self-hosting.md index 8f2d003..801ad43 100644 --- a/docs/operations/self-hosting.md +++ b/docs/operations/self-hosting.md @@ -4,37 +4,49 @@ This runbook owns initial single-host service setup. Application releases use [deployment](../deployment.md); accepted environment values and ports are in [configuration reference](../reference/configuration.md). -## Topology +## Embedded Media Configuration + +The current source embeds STUN and SFU in the Screener process under +[ADR-0013](../adr/0013-embedded-node-local-media.md). Hosted `STUN_URLS` +advertises ordinary discovery and enables the UDP 3478 listener on +`STUN_LISTEN_HOST` (IPv4, default `0.0.0.0`). Enabling +`NAT_PREDICTION_ENABLED` also binds UDP 3479/3480 on that address. Local and +public-link Client modes use discovery without creating media-service listeners. + +Set `SFU_UDP_PORT=7882` to enable automatic embedded SFU fallback. Its IPv4 bind +address is `SFU_LISTEN_HOST`, default `0.0.0.0`; `SFU_PUBLIC_IP` optionally +overrides the advertised IPv4 address when the host is behind NAT. Unset or +blank `SFU_UDP_PORT` keeps SFU disabled. SFU control uses authenticated Screener +signaling through the Web origin. No separate media-service executable, control +origin, or API credentials are needed by this source contract. + +These templates describe the embedded deployment. Its first cutover requires +the active-session check, matching artifacts and public media verification below. + +## Single-Host Topology ```text Browser -- HTTPS/WSS --> nginx :443 --> Screener :8787 -Browser -- /rtc/v1 WSS --> nginx :443 --> LiveKit :7880 Browser <---------- DTLS-SRTP P2P ----------> Browser -Browser -------- ordinary STUN/UDP --------> coturn :3478 -Browser <--------- DTLS-SRTP/UDP ----------> LiveKit :7882 -Screener -------- private RoomService -----> LiveKit :7880 +Browser -------- ordinary STUN/UDP --------> Screener :3478 +Browser -------- optional STUN/UDP --------> Screener :3479/3480 +Browser <--------- DTLS-SRTP/UDP ----------> Screener :7882 ``` -The tracked public baseline uses nginx, a valid Web TLS certificate, time -synchronization, coturn 4.17.2 or a newer patched release, -and LiveKit Server 1.13.6. LiveKit is required for the accepted SFU fallback -and tracked release wrapper. Keep the accepted patch pinned until its -replacement passes the same Browser/SFU acceptance boundary. +The public baseline uses one Screener Go binary, nginx, a valid Web TLS +certificate, and time synchronization. nginx terminates Web HTTPS/WSS and +proxies only to Screener's private application listener; media UDP reaches the +Go process directly. There is no separate SFU token endpoint or TCP 7880 service. Web and STUN may use different DNS names on the same public IP. Another proxy or -P2P-only service shape needs its own matching updater and recovery checks. +firewall owner needs its own matching updater and recovery checks. Expose only the public listeners in -[configuration reference](../reference/configuration.md). Screener 8787 and -LiveKit 7880 stay private. +[configuration reference](../reference/configuration.md). Screener TCP 8787 stays +private. STUN answers Binding requests only; TURN allocations remain unavailable. -## Tracked Templates +## Templates - [Screener systemd unit](../../deploy/systemd/screener.service.example) -- [LiveKit systemd unit](../../deploy/systemd/livekit.service.example) -- [LiveKit readiness drop-in](../../deploy/systemd/screener-livekit-readiness.conf.example) -- [coturn systemd unit](../../deploy/systemd/coturn.service.example) -- [STUN-only coturn config](../../deploy/coturn/turnserver.conf.example) -- [LiveKit config](../../deploy/livekit/livekit.yaml.example) - [nginx site](../../deploy/nginx/share.bonfire.icu.conf.example) Copy templates outside the repository, inject independent secrets through an @@ -44,8 +56,8 @@ domains, certificates, users, and resource limits required by the host. ## Initial Setup 1. Configure DNS and TLS for the Web origin and STUN name. -2. Install nginx, coturn, and the pinned LiveKit Server patch. The Screener - release is one static binary and needs no language runtime. +2. Install nginx. The Screener release is one static binary and needs no + language runtime or separate media-service executable. 3. Create an unprivileged `screener` service account, `/opt/screener/releases`, `/opt/screener/uploads`, and an access-restricted environment file. 4. Provision one verified initial immutable release and atomically point @@ -55,73 +67,140 @@ domains, certificates, users, and resource limits required by the host. 5. When stable room authority is enabled, use the service-owned `/var/lib/screener` StateDirectory and include the SQLite file in the host's state backup/recovery policy. -6. Install the tracked service and proxy templates. Bind Screener to loopback on - bare metal; containers may bind `0.0.0.0` only when publishing/firewall rules - preserve the same boundary. +6. Install the tracked service and proxy templates. Bind HTTP to loopback on + bare metal; STUN and SFU retain their independent public IPv4 binds. + Containers may bind HTTP to `0.0.0.0` only when publishing/firewall rules + preserve the private application boundary. 7. Apply the public-port allowlist and confirm every retired TURN/TCP port is closed. -8. Start coturn and LiveKit before Screener, then nginx. The LiveKit - readiness drop-in waits for its private control listener before Screener. +8. Start Screener, then nginx. Screener binds its configured media listeners + before accepting signaling and closes them with the application; a listener + bind failure rolls back startup. Verify public UDP reachability separately. -### First Go service cutover +## Optional Container -The tracked unit starts the Go binary. A Node-to-Go deployment is a one-time -infrastructure transaction, not a routine application update: its failure path -must restore the old unit and environment before starting the prior release. -Prepare and verify that recovery before installing the unit: +The [runtime-only Dockerfile](../../deploy/container/Dockerfile) consumes the +existing linux/amd64 Server release; it does not rebuild Go or Browser assets. +Its pinned [Distroless static non-root base](https://github.com/GoogleContainerTools/distroless) +includes CA certificates for HTTPS release checks and has no shell or package +manager. This is an optional recipe, not a published image or automatic installer. +On the local Linux Docker runtime, non-root/read-only startup, both persistence +modes, diagnostics export, STUN Binding and media listener cleanup pass. That +check covers the container lifecycle, not public-network SFU media or the host's +proxy/firewall configuration; verify those on the target deployment. + +Verify the release archive against its descriptor and manifest using the +[release procedure](../deployment.md), then extract its four files into a new +build-context directory: `screener-server`, `LICENSE`, `THIRD-PARTY-NOTICES.txt` +and `REVISION`. Use that directory, not the repository or a secrets directory: ```sh -sudo install -o root -g root -m 0644 \ - deploy/systemd/screener.service.example \ - /etc/systemd/system/screener.service -sudo systemctl daemon-reload -sudo systemctl show screener.service -p ExecStart -p Environment --no-pager +docker build --platform linux/amd64 \ + -f /path/to/Screener/deploy/container/Dockerfile \ + -t screener: /path/to/extracted-runtime +docker run --rm --env-file /path/to/screener.env \ + screener: --check-config ``` -The output must show `/opt/screener/current/screener-server` and -`SCREENER_ENV=production`. The staged environment removes `NODE_ENV` and -`PEER_ASSISTED_MEDIA`, and replaces route `NODE_DEBUG` with `SCREENER_DEBUG=route`. -Keep credentials, ports, capacity and media configuration unchanged. Do not -restart the old release using the Go unit. The routine release wrapper owns -only application-symlink recovery; it does not install or restore systemd units. +Use the existing [production configuration](../reference/configuration.md). +Keep `LISTEN_HOST=0.0.0.0` inside the container and retain the host's existing +HTTPS/WebSocket proxy. With Docker port mapping, set `SFU_PUBLIC_IP` to the +reachable server IPv4 address when SFU is enabled; container-private candidates +are not Internet-reachable. Advertised STUN names must resolve to the server's +public address, and UDP must reach the container directly. -The first-cutover transaction retains the previous unit, environment and closed -SQLite files, applies unit/environment plus release together, and restores those -exact values on failure before reloading systemd and restarting Node. After a -successful Go cutover, later application releases use the ordinary wrapper. +The following example publishes the standard listeners and keeps HTTP private +to a proxy running on the same host. Omit UDP publish options for disabled SFU +or auxiliary STUN listeners; use the matching port if configuration changes it. -LiveKit must be dedicated to this Screener application, set -`room.auto_create: false`, and expose its control listener only to the proxy and -application host. Screener binds its own listener before touching LiveKit, -rejects foreign room names, removes stale managed rooms, confirms the namespace -empty, and only then installs signaling. A competing application process must -make no LiveKit mutation. +```sh +docker run -d --name screener --restart unless-stopped --stop-timeout 20 \ + --read-only --cap-drop ALL --security-opt no-new-privileges \ + --env-file /path/to/screener.env \ + --mount type=volume,source=screener-data,target=/home/nonroot \ + -e ROOM_DATABASE_PATH=/home/nonroot/rooms.sqlite \ + -p 127.0.0.1:8787:8787/tcp \ + -p 3478:3478/udp -p 3479:3479/udp -p 3480:3480/udp -p 7882:7882/udp \ + screener: +``` -Coturn runs `stun-only`, `no-tcp`, and `no-tls`. It must answer STUN binding and -reject TURN allocation. Avoid verbose STUN/ICE logging because infrastructure -necessarily observes client IP addresses. +The image's `/home/nonroot` is owned by UID/GID `65532:65532` with mode `0700`. +A new named volume inherits that directory; an existing volume or bind mount +must already be writable by that identity. Keep this volume when replacing the +container. Omit `ROOM_DATABASE_PATH` for memory-only room authority. Diagnostics +remain opt-in and use `/home/nonroot/logs`; [export and retention](../reference/configuration.md#diagnostics) +remain the operator's responsibility. Do not mount application files writable. +With diagnostics enabled, `docker kill --signal=USR1 screener` requests a local +export without stopping the container; the ZIP remains in the mounted log directory. + +Check `/healthz` from the host and public proxy, then verify configured STUN/SFU +UDP and room/media behavior using [operational verification](#operational-verification). +There is no shell-based healthcheck in the image. Retain the previous image, +environment and volume backup before replacement. Stop the old container before +reusing its listeners or SQLite volume, and recreate with the previous image and +environment if verification fails. The bare-metal `release-app.sh` wrapper does +not manage containers. The first move from external media services still follows +the coordinated cutover below; this recipe changes no proxy or firewall itself. + +## Coordinated Embedded-Media Cutover + +Replacing the external STUN/SFU services, including Node-to-Go where still +needed, is one explicitly authorized infrastructure and protocol transaction. +The routine application wrapper cannot perform or recover it. + +1. Prepare and verify matching `screener-v22` Web/Server and native protocol v9 + Client artifacts. Check active sessions and obtain the owner's acceptance of + share interruption before cutover; old pages and Clients must reload or + update together. +2. Retain the exact prior release path, Screener unit and drop-ins, environment, + nginx configuration, and external media-service units, configuration, secrets, + and enabled/active state for rollback. Keep this material access-restricted + outside the repository. Define recovery for any persistent-state or firewall + surface the transaction actually changes. +3. Stage the current unit with + `ExecStart=/opt/screener/current/screener-server` and + `SCREENER_ENV=production`. Remove retired `NODE_ENV`, `PEER_ASSISTED_MEDIA` + and external `LIVEKIT_*` configuration; use the current + [environment contract](../reference/configuration.md), including + `SFU_UDP_PORT=7882` when fallback is enabled. Replace route `NODE_DEBUG` with + `SCREENER_DEBUG=route` when needed. Validate the new binary's production + configuration as the service user before switching. +4. Stop Screener and the old coturn/LiveKit services. Disable those old media + services and remove the installed Screener LiveKit-readiness drop-in so they + cannot reclaim ports or delay startup. Verify that UDP 3478/3479/3480/7882 + are released before the new Go process binds them. Install the current unit, + environment and nginx template, remove the old RTC proxy locations, validate + nginx, and reload systemd. Preserve unrelated proxy headers, limits, TLS, + asset handling, and firewall rules. +5. Atomically point the current release at the verified candidate, start + Screener, and reload nginx. Verify the actual unit command and environment, + local/public health, the matching Web asset, UDP listener ownership, public + STUN Binding and admitted SFU media, and the agreed active-session outcome. +6. On failure, stop the new Go process and confirm it released its UDP sockets. + Restore the exact prior application path, unit/drop-ins, environment, proxy + and changed infrastructure configuration; reload systemd and validate nginx. + Restore the previous external service states before starting the previous + application with its matching unit, then prove health and media. Restore + matching pages and Clients as part of protocol rollback. A symlink-only + rollback cannot recover this transaction. + +Retain the recovery material through the agreed acceptance window. This is a +single replacement contract with no permanent legacy aliases or migration mode. +Later application-only releases use the ordinary +[release wrapper](../deployment.md#atomic-cutover). ## Optional NAT Prediction -Set `NAT_PREDICTION_ENABLED=true` only while both auxiliary STUN-only listeners -are reachable. Add the following to the -service-owned coturn configuration, using the local address already bound by -the main listener: - -```ini -aux-server=:3479 -aux-server=:3480 -``` - -Keep `stun-only`, `no-tcp`, and `no-tls` unchanged. Allow UDP 3479 and 3480 in -both the cloud security group and the existing nftables input rule, validate -the complete ruleset with `nft -c -f /etc/nftables.conf`, then restart only -`coturn.service`. Verify `ss -lunp` shows all three UDP listeners and that an -ordinary STUN Binding succeeds on each. Do not add these URLs to production -`STUN_URLS`; enabled Browser connections derive them from the ordinary STUN -authority. Keep both listeners and firewall rules for as long as the capability -is enabled. Set `NAT_PREDICTION_ENABLED=false` before removing both -`aux-server` lines and both firewall rules, validate again, and restart coturn. +Allow UDP 3479 and 3480 in both the cloud security group and the existing +nftables input rule, validating the complete ruleset with +`nft -c -f /etc/nftables.conf`. Set `NAT_PREDICTION_ENABLED=true` and restart +Screener within an accepted service window. Verify `ss -lunp` shows all three +Screener STUN listeners and that a public Binding request succeeds on each. +Do not add auxiliary URLs to `STUN_URLS`; enabled connections derive them from +the ordinary STUN authority. Keep both firewall rules while the capability is +enabled. Set `NAT_PREDICTION_ENABLED=false` and restart Screener before removing +the auxiliary firewall rules. Avoid verbose STUN/ICE logging because those +requests contain client IP addresses. These same-IP auxiliary ports measure destination-port mapping behavior. They are not a full RFC 5780 alternate-address deployment; that requires a second @@ -130,13 +209,13 @@ public IPv4. ## Operational Verification - `/healthz` returns `{"status":"ok"}` locally and through the public origin. -- Screener, nginx, coturn, and LiveKit are active with expected restart - counts and bounded logs/resources. +- Screener and nginx are active with expected restart counts and bounded + logs/resources; Screener owns the configured STUN/SFU UDP listeners. - The official Trickle ICE sample obtains an ordinary UDP `srflx` candidate from the configured STUN service and no relay candidate. - A direct room presents media on two networks when Peer ICE succeeds. - A constrained route creates one managed Host publication and only admitted - subscriptions over LiveKit UDP; Peer descendants remain ordinary P2P. + subscriptions over embedded SFU UDP; Peer descendants remain ordinary P2P. - Blocking all UDP reaches a clear bounded failure. - Restarting Screener in stable mode retains room authority while Browser sessions and media reconnect from fresh process state. diff --git a/docs/product/media-quality.md b/docs/product/media-quality.md index 0fd8244..312db0a 100644 --- a/docs/product/media-quality.md +++ b/docs/product/media-quality.md @@ -3,8 +3,11 @@ This file owns the current Browser and Client media contract. Detailed measurements and platform limits live in [realtime quality research](../research/realtime-quality-adaptation.md) and [screen-audio research](../research/browser-screen-audio-quality.md). -[ADR-0007](../adr/0007-path-isolated-representation-quality.md) owns the SFU -adaptation decision. +[ADR-0007](../adr/0007-path-isolated-representation-quality.md) owns Browser +source intent and codec selection; [ADR-0013](../adr/0013-embedded-node-local-media.md) +owns the current candidate's shared Native/SFU output model. This file describes +that source contract. [Status](../status.md) separates it from the last known +production behavior and remaining acceptance. ## Capture And Controls @@ -36,9 +39,14 @@ adaptation decision. - Native quality changes use the same room settings. The Client prepares a new platform capture/encoder generation, then replaces the old generation behind the same encoded source and PeerConnections; audio-only changes update the - current Opus encoder directly. Platform quality preference uses the hardware - encoder's standard quality-versus-speed hint, while Pion/WebRTC still own - transport estimation and route evidence. + current Opus encoder directly. A live profile update may remain pending while + the selected source produces no frames. Setup and encoding failures remain + bounded; quiet-source waiting must not block pause, stop or connection control. + Applied settings change only after the replacement is ready and installed; + failure preserves the previous capture and profile. Platform quality preference uses the hardware + encoder's standard quality-versus-speed hint. Pion owns Native WebRTC + transport; the shared media adapter uses LiveKit media components for bandwidth + estimation, forwarding allocation, pacing and recovery. Windows VP8 uses libvpx's realtime mode; it does not claim the same hardware quality-versus-speed control or per-edge Browser adaptation. - With Client available, Browser H.264 capture can use one local sender and the @@ -101,32 +109,77 @@ quality/source changes retain it. Other native platform encoders remain H264. ## Framework-Owned Adaptation -Each direct, relay, or SFU video sender owns one clone of its capture or received +Each Browser direct, relay, or SFU video sender owns one clone of its capture or received source track. The original track remains a source and local presentation track; it is never attached directly to an outbound sender. Replacing or retiring a sender also stops its clone, so native adaptation state cannot survive by being inherited through the original track. Host pause and live capture constraints are propagated to current Host-owned clones. -Each PeerConnection still owns stock WebRTC congestion control and sender +Each Browser PeerConnection owns stock WebRTC congestion control and sender adaptation. Clones share one underlying media source and therefore do not provide complete simultaneous isolation: framework source-wants aggregation may still partially reduce frames available to sibling clones. Sibling outputs may differ; -Screener does not impose a room-wide minimum. The Host's one SFU publication uses -the share-generation codec and selected ceiling but no application-defined -simulcast ladder or backup codec. Pinned LiveKit defaults construct -representations; Dynacast aggregates demand and server send-side BWE selects -subscriber forwarding. AdaptiveStream stays disabled because any Viewer may -relay its received track. +Screener does not impose a room-wide minimum. The Host's one Browser SFU +publication uses the share-generation codec and selected ceilings. Its ordinary +simulcast outputs follow pinned LiveKit screen-share construction: original and +half size at the same frame rate, with a quarter-bitrate lower ceiling and its +150 kbps floor. Tiny sources use one output. There is no backup codec or third +product preset. Browser WebRTC budgets the aggregate publication. + +The embedded SFU forwards an existing suitable encoding to each subscriber. +LiveKit media components own its per-subscriber bandwidth estimation, allocation, +RTP projection, pacing and loss recovery; the LiveKit room service and Browser +SDK are not used. Direct-subscriber demand controls the publisher's active +output prefix through the current room signal. Target, currently forwarded and +recovery-preparation demand stay distinct from actual delivery. Viewport size +does not choose a layer because a Viewer may relay its received media. Tracks received from a PeerConnection do not own capture constraints. Native capture therefore retains source resolution and frame-rate ownership across its -loopback Browser bridge; the existing Browser SFU publisher applies only sender -parameters and LiveKit's representation policy to that remote source. +loopback Browser preview. Native Host SFU publication uses the encoded source +directly and does not re-encode that preview through a Browser publisher. -Native sender edges normally reuse one encoded source. A Native Viewer forwards -compatible H.264/VP8 and Opus payload without decoding or re-encoding it; each outbound -PeerConnection owns its RTP identity and transport feedback. When an edge is +Native parents and the SFU share the media forwarding adapter. Each Native +parent considers only its direct children and its admitted SFU publication. +Compatible children reuse an available encoded output; a missing lower output +starts one shared decode/scale/encode group, while healthy forwarding retains +the original input without starting a decoder. The group produces the highest +needed output and its lower fallbacks and stops unused upper outputs. Each child +receives only its selected output. The forwarding SFU adds no server transcoder. + +Native capture uses the same screen-share output construction within Host +ceilings. Its source, source clock and higher outputs remain independent of +lowest-output bitrate adaptation. The SFU upstream request is the maximum of +its direct-subscriber demands; Native Host combines it with direct Peer demand, +while the publication still has one aggregate upload budget. Multiple outputs +have real encode and upload costs; this is not a one-encode guarantee. + +Packet identity and delivered counters belong to each outbound edge. Unused +outputs do not inflate that edge's FPS or bitrate. Bounded codec workers and +packet queues isolate slow children and retire with the source generation. +Windows shared local outputs use one complete stock WebRTC encoding/adaptation +pipeline each, including source restrictions, rate correction and resource +feedback. Resolution/FPS adaptation keeps the capture and transport identities; +actual output dimensions update forwarding metadata without resetting sibling +stream trackers. H264 uses native NV12/MFT and WebRTC's parsed-QP adaptation; +VP8 uses the library encoder. Other platform producers retain their current +fixed-output implementation pending equivalent physical acceptance. +Each source groups compatible effective direct-child requests and keeps +incompatible weak budgets independent. Per-group rates change in place; joining +another group's output waits for accepted recovery and keeps the existing +PeerConnection, pacer and bandwidth estimator. Original received media stays +encoded, while a relay uses one decoder process for its independently activated +missing outputs. Source-generation retirement rejects old frames before they +can change attachment. Private physical slots are bounded by admitted consumers, +not fixed spatial presets, and unused codec workers retire independently. +This does not establish full Browser parity, hardware-overload acceptance or +seamless adaptation at extremely low bandwidth. + +Native sender edges normally reuse the shared encoded source. A Native Viewer +forwards suitable H.264/VP8 and Opus payload unchanged and derives only a missing +lower video output; each outbound PeerConnection owns its RTP identity and +transport feedback. When an edge is persistently degraded, the existing quality operation may prepare a Browser WebRTC sender from the stable local bridge as that edge's candidate. The old edge stays live until the Viewer proves the candidate is better; a failed @@ -148,6 +201,10 @@ disabled; this accepts less burst-loss resilience in exchange for bounded publisher traffic. The Browser and source still decide whether an audio track exists and what is actually delivered. +A Native audio-process failure does not end healthy video. A later explicit +source replacement resumes audio through the existing track and encoder owner; +it does not trigger an automatic capture retry or change room audio topology. + ## Observable Truth Requested resolution, frame rate, bitrate, content intent, codec, and audio @@ -156,11 +213,18 @@ are the observable result. Connection detail may show actual path, codec, resolution, FPS, bitrate, loss, jitter, RTT, and encode/decode behavior when the Browser exposes them; missing or reset counters remain unknown. -WebRTC and LiveKit remain the media-adaptation owners. ADR-0005's deployed -native-edge convergence consumes only a persistent categorical sender limitation +Browser WebRTC and the shared Pion/LiveKit media adapter own media adaptation. +ADR-0005's native-edge convergence consumes only a persistent categorical sender limitation and proof from the same Viewer over a real prepared candidate; it does not turn quality metrics into media settings or a weighted/global route score. +Native categorical evidence includes the media framework's committed allocation +deficiency: fitting a reduced output into the available budget does not clear a +continuing framework limitation. Publication evidence considers only requested +encodings, so an intentionally inactive encoding is not a bandwidth failure. +Missing feedback or frame progress remains unknown. This category neither +infers a physical bottleneck nor ranks routes by configured quality ceilings. + Web pages cannot guarantee Host background capture, mobile background playback, or relay execution after Browser/OS suspension. Those remain physical lifecycle boundaries, not keepalive features. diff --git a/docs/product/presentation-lifecycle.md b/docs/product/presentation-lifecycle.md index 4e1ccae..b17d108 100644 --- a/docs/product/presentation-lifecycle.md +++ b/docs/product/presentation-lifecycle.md @@ -103,14 +103,21 @@ black screen, ICE-connected state, or unproved `playing` event. before showing a retained frozen background and recovery or failure state. - Room not found/expired is distinct from current-room access denial and does not prompt for a password. -- Manual reconnect rebuilds the same route. It does not select a new parent. +- Manual reconnect recovers the same route. It does not select a new parent. - An actionable overlay may appear while fullscreen remains active; application state never forces the user out of fullscreen. Visibility, page freeze, and pagehide suppress application decoded-stall authority. Returning to the page rebaselines time and proves a current frame -again before clearing recovery. LiveKit page-leave auto-disconnect is disabled, -but Browser or OS suspension and page reclamation remain outside Web guarantees. +again before clearing recovery. SFU media is retained independently of transient +room-signaling loss, but Browser or OS suspension and page reclamation remain +outside Web guarantees. + +Failure of the local Native media bridge disables Native reception for the +current Viewer session before route recovery runs. Future peers in that session +use Browser reception; existing healthy peers are not torn down by this choice. +Parent-network failure alone is not evidence that the local Native capability +is unavailable. ## Presence And Diagnostics diff --git a/docs/product/rooms-access.md b/docs/product/rooms-access.md index 38db458..4200578 100644 --- a/docs/product/rooms-access.md +++ b/docs/product/rooms-access.md @@ -11,6 +11,14 @@ the storage decision; implementation detail belongs in code and tests. a room; it is not a secret or permanent identity. - Every room has a 256-bit Host token. The server stores only its digest; the owning Browser keeps the raw token locally. +- Each Host tab retains its current room in session storage. The origin keeps + one persistent resume hint and a separate preferred code; a new tab may resume + that room only when no other tab claims it. Secure Browser origins use Web + Locks, including protection against copied tab storage. Without Web Locks, + fresh tabs create independent rooms and same-tab reload can retain authority, + but duplicated-tab exclusion is not guaranteed. Closing the tab releases its + claim; losing Host authority clears that tab without erasing another tab's + resume hint. - Dormant rooms have a configurable lease, default 24 hours. An authenticated online Host clears the lease deadline whether or not it is currently sharing; Host absence starts the dormant lease. The exact Host token resumes an @@ -32,7 +40,10 @@ Three independent authorities exist: authorized page renews it through the site-access check while it remains active. It authorizes room creation, Host role, and code-only Viewer attempts; it is not a room credential, and Viewer-grant admission cannot - create or renew it. + create or renew it. Cookie transport attributes follow the configured + destination: public HTTPS retains the Secure host-prefixed cookie, while a + separately configured HTTP LAN origin can use its HTTP cookie. Login, + renewal, room creation and WebSocket admission use that same selection. 2. **Host ownership.** The exact Host token authorizes that room's Host and access-management operations. It cannot authorize another room. 3. **Viewer invitation.** Every room creates a 128-bit, 22-character base64url @@ -60,10 +71,10 @@ remain separate. Rotating or revoking the Viewer grant is a strong authorization change. The new generation is committed before old Viewers, signaling grace, routes, and sender -edges are closed. An affected SFU Viewer is removed by its exact LiveKit -identity and confirmed absent. This does not revoke an already issued -self-hosted LiveKit token: its subscription stays charged until that publication -generation drains. Changing code-entry policy or password affects later +edges are closed. An affected SFU Viewer's exact embedded subscription is closed +before its egress reservation is released. Room-authenticated signaling cannot +recreate a revoked subscription; no separate media token survives revocation. +Changing code-entry policy or password affects later code-only attempts and does not silently revoke invitations. ## Persistence Modes diff --git a/docs/product/routing-transport.md b/docs/product/routing-transport.md index 96adaf1..7f66dfd 100644 --- a/docs/product/routing-transport.md +++ b/docs/product/routing-transport.md @@ -16,8 +16,8 @@ owns ports and [self-hosting](../operations/self-hosting.md) owns service setup. - SFU-fed and peer-fed Viewers may both serve as ordinary peer parents. - A route ends in usable media, explicit bounded waiting, or clear failure. -Every deployment uses this controller. Omitting LiveKit keeps the same bounded -P2P graph without its SFU fallback; there is no separate Host-star mode. +Every deployment uses this controller. Leaving embedded SFU disabled keeps the +same bounded P2P graph without its SFU fallback; there is no separate Host-star mode. ## Per-Share Route Policy @@ -111,32 +111,44 @@ fanout skew; it is not periodic balancing and does not move a healthy branch without admitted overlap. Framework reconnect runs before route reassignment. Manual media reconnect also -rebuilds only the current P2P parent or current SFU subscription; it does not +recovers only the current P2P parent or current SFU subscription; it does not perform quality selection or choose another route. ## SFU Fallback -The only application fallback is one dedicated LiveKit SFU publication from the +The only application fallback is one admitted embedded SFU publication from the Host, with independently admitted Viewer subscriptions. The controller may create it through the same bounded transition when Host capacity is occupied. No Viewer creates a second publication. -Browser SFU PeerConnections use LiveKit-signaled UDP candidates and no external -ICE-server list. Ordinary peer connections use deployment STUN. Screener +SFU offer/answer, ICE and output demand use the existing authenticated room +WebSocket. The server binds each physical connection to the current participant, +share and publication; it issues no separate media token or room-service URL. +Browser SFU PeerConnections use the server's UDP candidates and no external +ICE-server list. Ordinary peer connections use deployment STUN. Hosted Screener +owns its configured Binding-only STUN listeners and optional SFU UDP listener in +the same Go process. Screener configures no TURN, ICE/TCP, media TCP, or TLS-relayed media path. HTTPS/WSS is a separate control transport and remains TLS/TCP. SFU resources remain bounded throughout reservation, use, and cleanup; stale -credentials cannot recreate off-ledger media. Grant revocation, Viewer leave, -and SFU-to-P2P replacement remove the exact LiveKit Viewer and confirm absence, -but the subscription remains charged until its publication generation drains. -ADR-0005 owns that lifecycle. +signaling cannot recreate off-ledger media. Grant revocation, Viewer leave, +and SFU-to-P2P replacement close the exact physical subscription before releasing +its egress reservation. Publication teardown closes its ingress and descendants +before releasing their reservations. There is no outstanding external token to +retain after those resources close. ADR-0005's graph and operation rules remain; +[ADR-0013](../adr/0013-embedded-node-local-media.md) replaces the external-service lifecycle. + +Transient room-signaling loss does not close healthy SFU media. The current +physical connection can restart ICE; actual terminal media failure returns to +the same bounded route recovery. Source capture remains Host-owned throughout. ## Quality And Privacy Boundaries -WebRTC and LiveKit own ICE, consent, congestion control, bitrate, frame rate, -resolution, retransmission, reconnect, and SFU layer selection. When the -per-share convergence gate is enabled, Screener uses an exact persistent native +Browser WebRTC and the shared Pion/LiveKit media adapter own ICE, consent, +congestion control, bitrate, frame rate, resolution, retransmission, pacing and +layer selection. Direct-child media demand does not create a topology operation. +When the per-share convergence gate is enabled, Screener uses an exact persistent native sender limitation only to trigger one measured experiment through the existing serial operation. Three fresh complete limited deltas from one exact sender may trigger that experiment on a newly committed availability or direct-convergence @@ -180,11 +192,12 @@ One-link mode tunnels only HTTP/WebSocket control and adds public STUN to the same Browser/Native P2P edges. Reachable peers form one mixed relay tree; an unreachable path adds no route type or score. Site mode uses Site transport. -A native Host reserves one loopback edge for Browser preview and the existing -SFU publisher; it consumes no route-copy capacity or gateway mapping. Direct -children share the native encode, while assigned SFU keeps the Browser LiveKit -representation and recovery policy. +A Native Host reserves one loopback edge for Browser preview; it consumes no +route-copy capacity or gateway mapping. Direct children and the admitted Native +SFU publisher reuse the encoded source through the shared media adapter. The SFU +publication still consumes one route copy and one aggregate upstream budget. A native Viewer terminates only its P2P upstream, reserves one Browser playback -edge, and reuses compatible H.264/Opus for bounded children. Browser keeps SFU, +edge, and reuses compatible H.264/VP8/Opus for bounded children, deriving a missing +lower output only on direct-child demand. Browser keeps SFU subscription, route revisions, frame proof, and fallback. diff --git a/docs/project-memory.md b/docs/project-memory.md index a33ed4b..a4c04d2 100644 --- a/docs/project-memory.md +++ b/docs/project-memory.md @@ -1,6 +1,6 @@ # Project Memory -Last updated: 2026-09-06 +Last updated: 2026-09-08 Screener is private, low-latency game screen sharing for one Host and up to 20 authenticated friends. The current product surface is Web Host, Web Viewer, and @@ -13,26 +13,26 @@ Site through the system Browser. Its single Go process presents Local, temporary public-link, and saved-Site choices on every launch, starts one process-level loopback capability service, owns the Local room authority in-process when selected, and owns the explicit native media path chosen in the same Web UI. -Room source and media -implementation are independent: a topology may mix Browser and Native peers -without changing participant, signaling, capacity, or route identity. Native -Host and H.264/VP8 Viewer receive/relay adapters share the same Pion media edge; -unsupported Viewer media falls back to Browser. A Client-assisted Browser Host -can feed that fanout through one local H.264 sender while quality convergence is -enabled; losing the Client preserves Browser capture. Local remains -a serverless-LAN mode by default; explicit one-link mode exposes that same -authority through a session-scoped public control tunnel while media remains -P2P. Minimal Windows, macOS, and Linux adapters terminate at one encoded-frame -boundary; only Windows has completed physical native-media acceptance, while -the other two compile and package. Native capture consumes the same room quality -settings and applies live quality or source changes behind its stable encoded -source and connections. Native P2P edges feed Pion GCC's categorical payload- -capacity result into the existing route evidence windows. Healthy Native sender -edges share one hardware encode; the existing quality operation may test one -stock Browser sender for a persistently degraded edge without lowering the -shared source. One reserved loopback edge gives the system Browser a preview and -lets native Host media reuse the existing LiveKit publisher when SFU fallback is -assigned; broader package acceptance remains a later gate. +Room source and media implementation are independent: a topology may mix +Browser and Native peers without changing participant, signaling, capacity, or +route identity. Local is LAN-first; explicit one-link mode exposes that same +authority through a temporary public control tunnel while media remains P2P. +Host authority belongs to a tab, with one separate persistent origin resume +hint. The Client admits two independent native control sessions under its one +loopback service. + +The current source embeds Binding-only STUN and optional SFU forwarding in +the Hosted Go process. Native and SFU use one shared Pion/LiveKit media adapter; +Native parents reuse suitable encoded outputs and derive a missing lower output +only for direct-child demand. Source-owned groups retain the highest needed +output and lower fallbacks, while each child receives its selected output. +Native Host SFU publication reuses the encoded source directly; its loopback +Browser edge owns preview only. Pure Browser capture and relay retain normal +WebRTC senders. Windows, macOS and Linux producers share one encoded-frame +boundary and Host quality controls. Source replacement retains route and +connection identity; lower-output adaptation preserves the original and higher +sibling outputs. [Status](./status.md) owns the remaining physical and release +acceptance boundary; runtime release descriptors own deployed identity. The Local launcher keeps an optional user-chosen site-access password; leaving it blank keeps the self-contained site open. @@ -62,15 +62,17 @@ implementation and routine UI detail. room-wide, connection-local NAT prediction; its Host switch defaults on when available, and anonymous provenance distinguishes ordinary from predicted selected paths. The configuration default remains stock ICE. The only - application fallback is one bounded LiveKit SFU/UDP + application fallback is one bounded embedded SFU/UDP publication; Screener configures no TURN or TCP media route. A pre-share peer-only policy can exclude that SFU suffix for one share generation. - Every endpoint shares one steady outbound-copy cap, default `2` and limited to `1..3`. One room controller owns one committed graph, one reconcile loop, and one serial child operation. First decoded frame commits availability work; accepted quality work adds native edge proof before commit in that operation. -- WebRTC and LiveKit own congestion control, media adaptation, reconnect, and SFU - layers. Current source defaults persistent native-edge local P2P convergence +- Browser WebRTC and the shared Pion/LiveKit media adapter own congestion control, + media adaptation, pacing and layer selection. Parent demand is local to direct + children and never a room-wide minimum or topology rebalance. Current source + defaults persistent native-edge local P2P convergence on, with a pre-share Host opt-out. SFU quality work is limited to multi-root Host fanout relief and each Viewer must prove its own candidate non-regression before commit. A persistently limited edge with no different Peer parent may @@ -87,11 +89,13 @@ implementation and routine UI detail. ## Current Snapshot -The audit candidate uses the strict `screener-v21` Browser/server contract with -committed media readiness, bounded candidate-relative progress and one-shot -Host-root convergence. It retains optional SQLite room authority and the -H.264/VP8 sender gate. [Status](./status.md) owns the compact current execution -snapshot rather than duplicating per-release identity. +The candidate uses strict `screener-v22` Browser/server signaling, Native control +v9 and capture v7 together. Embedded SFU SDP/ICE and demand travel over the +authenticated room WebSocket, without an external room service or media token. +Committed first-frame readiness, bounded candidate-relative progress, optional +SQLite room authority and the H.264/VP8 sender gate remain. [Status](./status.md) +separates this source from last known production; protocol rollback requires +matching Browser and Client artifacts, not only an application symlink. Current work is owned by the [TODO ledger](./todo.md). Open physical evidence is owned by [verification status](./verification-status.md). Environment and initial diff --git a/docs/reference/configuration.md b/docs/reference/configuration.md index 9448871..ffa822e 100644 --- a/docs/reference/configuration.md +++ b/docs/reference/configuration.md @@ -18,21 +18,22 @@ service secret store or an untracked access-restricted environment file. | `ROOM_DATABASE_PATH` | Optional absolute SQLite file path; unset selects memory mode. | | `MAX_VIEWERS_PER_ROOM` | `1..20`, default `8`. | | `ENDPOINT_MEDIA_COPY_CAPACITY` | Shared endpoint steady-copy cap `1..3`, default `2`. | -| `STUN_URLS` | Comma-separated `stun:` URLs; at least one is required in production. | -| `NAT_PREDICTION_ENABLED` | Optional bounded NAT prediction capability, default `false`; requires an ordinary `STUN_URLS` endpoint on UDP 3478 plus reachable same-host UDP 3479/3480 listeners. | +| `STUN_URLS` | Comma-separated advertised `stun:` discovery URLs; at least one is required in production. These are not local bind addresses and may use an unproxied DNS name separate from the Web origin. | +| `STUN_LISTEN_HOST` | Hosted IPv4 STUN bind address, default `0.0.0.0` when `STUN_URLS` is configured; independent of HTTP `LISTEN_HOST`. Local Client construction creates no STUN listeners. | +| `NAT_PREDICTION_ENABLED` | Optional bounded NAT prediction capability, default `false`; requires an ordinary `STUN_URLS` endpoint on UDP 3478. Hosted startup binds UDP 3479/3480 before advertising the capability. Firewall reachability remains an operator requirement. | -Automatic SFU fallback is enabled when all four values below are present: +Automatic SFU fallback runs inside the Hosted process when `SFU_UDP_PORT` is set: | Variable | Contract | | --- | --- | -| `LIVEKIT_URL` | Browser `ws:`/`wss:` origin with no path; production requires WSS. | -| `LIVEKIT_API_URL` | RoomService `http:`/`https:` origin with no path; production plaintext is loopback-only. | -| `LIVEKIT_API_KEY` | Independent LiveKit API key. | -| `LIVEKIT_API_SECRET` | Independent secret of at least 32 bytes. | +| `SFU_UDP_PORT` | Optional UDP media port `1..65535`; unset or blank disables SFU. Set `7882` for the standard public listener. | +| `SFU_LISTEN_HOST` | IPv4 bind address, default `0.0.0.0`; independent of HTTP `LISTEN_HOST`. Read only when SFU is enabled. | +| `SFU_PUBLIC_IP` | Optional explicit IPv4 advertised-address override for a host behind NAT. Read only when SFU is enabled. | -The site-access password, LiveKit API key, and LiveKit secret must not reuse one -another. LiveKit tokens are short-lived media credentials and do not provide -application E2EE. +SFU control uses the application's authenticated signaling connection. No +separate control origin or infrastructure credentials are configured. Local and +public-link Client construction create no SFU listener. The relay does not +provide application E2EE. Removed access, room TTL, endpoint-tier, room-rollout, and TURN variables fail startup even when blank. A present `NODE_ENV` fails the same way, so a stale @@ -40,23 +41,82 @@ environment file cannot silently drop a deployment out of production. The private deployment is upgraded atomically; there are no compatibility aliases or dual configuration readers. -During pre-release route canaries, `SCREENER_DEBUG=route` enables sanitized room -and participant-ordinal events. It records route reasons, candidates, revisions, -quality states and commit/failure outcomes, but not raw Peer IDs, SDP, ICE -candidates, tokens or media credentials. +## Diagnostics + +Client file diagnostics default off; enable them with `--debug` or +`SCREENER_DEBUG=client`. Structured lifecycle, build-revision, native-capability, +fixed native-failure and sanitized Local route events go to `client.log`. +Native share changes, validated capture states, connection states and selected +candidate types/provenance are included without source names or addresses. +Failed profile preparation records a fixed rejection stage. Native outbound +connections record ICE and DTLS state separately; local-bridge closure records +candidate type/transport counts, never candidate addresses or SDP. +The embedded Local server uses the same file. Debug events do not stream through +the TUI or change machine-readable stdout. On the Client, `SCREENER_DEBUG=route` +alone retains console route tracing and does not enable file capture or `D`. + +The default Client directory is `logs` beside the executable. If that directory +is not writable, the Client uses `Screener/logs` inside the OS user-cache directory +and displays the actual log path. `--log-dir ` overrides +`SCREENER_LOG_DIR`; either explicit choice must be writable or startup fails. +Selecting a directory alone does not enable diagnostics. + +Press `D` in the interactive Client terminal to export a ZIP without stopping +the Client. The terminal shows the saved path as a local-file hyperlink when +supported; the visible path can also be copied normally. English, Chinese and +visual modes share that action. Non-interactive and `TERM=dumb` sessions export +at orderly shutdown, since they do not read TUI keys. + +Hosted Server diagnostics use `--debug`, `SCREENER_DEBUG=server` or +`SCREENER_DEBUG=route`. They persist structured events in `server.log` while +normal service journal output continues. The directory is `SCREENER_LOG_DIR`, +otherwise the first systemd `LOGS_DIRECTORY`, otherwise `logs` under the working +directory. The tracked systemd unit supplies `/var/log/screener` with mode `0700` +through `LogsDirectory`, which remains writable with `ProtectSystem=strict`. +An unwritable selection fails startup. In a running debug-enabled +Unix server, `kill -USR1 ` saves a bundle and prints its path in the journal. +Every platform also exports at orderly debug shutdown. Windows does not have +the Unix signal action; forced termination cannot produce a shutdown snapshot. +No diagnostic HTTP listener or automatic upload is added. + +`--debug` enables the existing sanitized route events for the local Go process; +`SCREENER_DEBUG=route` also works when loaded from the Server `.env`. Those events +record room and participant ordinals, route reasons, candidates, revisions, +quality states and commit/failure outcomes. They exclude raw Peer IDs, SDP, ICE +candidates, tokens and media credentials. Raw third-party protocol logging is +not enabled or copied into these files. + +Each component retains an 8 MiB current JSON log and one 8 MiB previous log. +Exports contain those existing logs, build/platform metadata, Go `runtime.MemStats`, +a sampled Go heap/allocation profile (`heap.pprof`) and aggregate goroutine +stacks (`goroutines.txt`). These are runtime counters and allocation/stack +profiles, not raw process-memory dumps. They do not collect environment values, +configuration files, credentials, media payloads, Browser logs or memory from +the separate C++ capture process. Exported ZIPs remain in the selected directory +until the user or operator removes them; log rotation does not delete exports. + +For Browser diagnostics, add `?debug=1` to the page before reproducing the +problem. The page retains its last 256 bounded events and exposes +`window.__SCREENER_DEBUG__.export()` for manual JSON export from DevTools. +The report includes the Browser asset name and signaling contract, capture +exception categories, capture-setting outcomes, signaling state and native +request/state outcomes. The Native bridge's failure snapshot includes ICE/DTLS +states, live receiver counts and candidate type counts without delaying teardown. +It excludes raw exception messages, credentials, invitation fragments, URLs, +SDP, ICE candidates and media payloads. Collection +stays in the current page; there is no automatic upload or persistent log. ## Public And Private Ports | Port | Scope | Owner | | ---: | --- | --- | | TCP 80/443 | public | HTTP redirect and HTTPS/WSS reverse proxy | -| UDP 3478 | public | STUN-only coturn | -| UDP 3479/3480 | public when NAT prediction is enabled | auxiliary STUN-only coturn listeners | -| UDP 7882 | public when SFU enabled | LiveKit WebRTC media | +| UDP 3478 | public | in-process STUN-only Screener listener | +| UDP 3479/3480 | public when NAT prediction is enabled | in-process auxiliary STUN-only Screener listeners | +| UDP 7882 (or `SFU_UDP_PORT`) | public when SFU enabled | in-process Screener WebRTC media | | TCP 8787 | private | Screener application | -| TCP 7880 | private when SFU enabled | LiveKit signaling/control | -TCP 3478, TCP/TLS 5349, TURN relay ranges, LiveKit media TCP, and other media +TCP 3478, TCP/TLS 5349, TURN relay ranges, media TCP, and other media ports remain closed. HTTPS/WSS transport is independent of the UDP-only media contract. @@ -64,10 +124,14 @@ When `NAT_PREDICTION_ENABLED=true`, the server derives `stun::3479` and `:3480` from the first ordinary STUN authority on UDP 3478. The Host sees a pre-share switch that defaults on and may disable it. The capability adds no media route or third-party service. It needs both -cloud security-group rules and the host's `/etc/nftables.conf` rule; coturn must -bind both auxiliary listeners with `aux-server`. Opening a cloud port without a -listener has no effect, and no new DNS record is required. Disable the -capability before removing either listener or firewall rule. Same-IP ports +cloud security-group rules and the host's `/etc/nftables.conf` rule. Screener +binds every required UDP listener before opening room persistence or accepting +signaling; a bind failure rolls back all newly owned sockets. Close and End +retire these listeners with the application. Existing coturn listeners must +be retired in the coordinated deployment because two processes cannot own the +same ports. Opening a cloud port without a listener has no effect, and local +binding alone does not prove external reachability. Disable the capability +before removing firewall rules. Same-IP ports expose destination-port allocation behavior; a full RFC 5780 alternate-address test requires a second public IPv4. @@ -80,5 +144,5 @@ test requires a second public IPv4. - The four-digit code space fixes the maximum managed Host publications at 9,000; maximum subscription admission is that capacity times the configured per-room Viewer limit. These are admission bounds, not throughput claims. -- One application process owns room authority and LiveKit admission. Multiple +- One application process owns room authority and SFU admission. Multiple processes require a new shared atomic owner. diff --git a/docs/research/advanced-peer-distribution.md b/docs/research/advanced-peer-distribution.md index 21d9eec..2fedc98 100644 --- a/docs/research/advanced-peer-distribution.md +++ b/docs/research/advanced-peer-distribution.md @@ -1,6 +1,6 @@ # Advanced Peer Distribution -- Last reviewed: 2026-08-27 +- Last reviewed: 2026-09-08 - Scope: advanced low-server distribution candidates for sub-second interactive media - Status: research comparison and falsifiable gates, not an accepted product route @@ -60,7 +60,7 @@ compute, not total room upload. The Host sends complementary stripes through two deterministic, interior-node-disjoint trees. An internal node forwards only its stripe; a Viewer needs both parents and a bounded assembler. SVC alone cannot supply this shape: -its layers remain in one encoded RTP stream, and Encoded Transform does not move +its layers remain in one encoded RTP stream, and standard Encoded Transform does not move encoded frames between transports. Start with 50/50 single-layer frame/object striping. Retain only if: @@ -116,6 +116,148 @@ same egress class as an SFU. Keep it only if equal-quality measurements improve latency or server CPU by at least 20% at the same egress, and draft-version churn is isolated behind a small adapter. +## Chromium Legacy Fanout Probe + +On 2026-09-08, an isolated installed Chrome 152 session demonstrated one Browser +encoder feeding two PeerConnections through legacy `createEncodedStreams()`. +The [manual probe](../../scripts/browser-encoded-fanout-probe.mjs) serves only a +synthetic 640x360 canvas over loopback HTTP; it uses neither Client nor SFU. +No browser feature flags were added to permit encoded-frame forwarding. + +The secondary sender has a non-updating canvas track to establish its RTP path. +Each primary encoded frame is cloned into both writable streams. With the +secondary connection established before primary encoding starts, VP8 encoded +208 frames once and both receivers decoded 208 at about 30 fps. H264 encoded +206 frames once and both receivers decoded 206. The secondary reported zero +encoded frames and zero encode time in both runs. This is clean-path frame +reuse, not whole-process CPU savings or hardware-acceleration evidence. + +It has three concrete failures/limits: + +- The secondary sender's `framesSent` stayed zero despite sending the complete + payload. Its `qualityLimitationReason=none` is not useful proof that the + borrowed encoding fits that connection. Existing Screener sender evidence + cannot consume those counters unchanged. +- In a VP8 late-join arm, the secondary received bytes but decoded zero frames + and sent 34 PLI requests. Those requests did not make the primary encoder + produce a recovery frame. Merely copying frames does not connect all feedback. +- With secondary `maxBitrate=50000` confirmed by parameter readback, it still + sent 387,287 payload bytes in the approximately seven-second VP8 window, + roughly 440 kbps, and both receivers decoded 207 frames. Borrowed output + bypasses that sender's encoder-rate ceiling. This is a parameter-constraint + check, not network shaping or evidence that transport BWE itself is broken. + +This exception does not overturn the [standard owner/counter restrictions](https://www.w3.org/TR/2026/WD-webrtc-encoded-transform-20260625/#stream-processing). +Pinned Chromium [legacy sender construction](https://chromium.googlesource.com/chromium/src/+/refs/tags/152.0.7977.82/third_party/blink/renderer/modules/peerconnection/rtc_rtp_sender.cc) +uses the underlying sink's legacy constructor, while the +[sink](https://chromium.googlesource.com/chromium/src/+/refs/tags/152.0.7977.82/third_party/blink/renderer/modules/peerconnection/rtc_encoded_video_underlying_sink.cc) +applies those checks when restrictions are enabled. Its +[transformer](https://chromium.googlesource.com/chromium/src/+/refs/tags/152.0.7977.82/third_party/blink/renderer/platform/peerconnection/rtc_encoded_video_stream_transformer.cc) +can deliver a written frame to the sole registered sink. No metadata rewriting +or disabled-standard-check browser flag was needed by this legacy probe. + +Therefore "pure Browser reuse is impossible" is too broad. A current Chromium +legacy path is executable; a portable, correctly adapting production adapter is +not established. The standard transform is after encoding, not an encoder +factory injection API. Unproven bandwidth/probe behavior, PLI routing, truthful +stats, non-Chromium compatibility and API lifetime remain reasons not to ship +this shortcut. No fallback switch, polling quality controller or transport +replacement was added to the product. [Results](./data/browser-encoded-fanout.json) +retain only selected public counters, not candidates, addresses or frame data. + +Run `node scripts/browser-encoded-fanout-probe.mjs`, optionally `--h264`, +`--late-join` or `--limit-secondary`, and open the printed loopback URL in an isolated Chrome session. +The page closes its PCs/tracks and posts the result; the server exits on result +or at its 60-second deadline. Output stays under ignored `build/encoder-pool`. +These observations are not an assertion that every arm should decode: late join +is the explicit negative control. + +### September 9 Continuation Plan + +The owner reopened Browser reuse after the Native/service checkpoint. Keep this +experiment separate from product startup, routing and existing Native owners. +Evaluate the smallest encoded-frame adapter before considering a new pipeline: + +1. Reproduce ordinary two-sender and shared-frame controls on the same H264/VP8 + source. Count actual encode work, decoded frames and transport bytes separately. +2. Test late join with one supported sender keyframe request and a recovery-frame + admission boundary. Do not add periodic keyframes, replay an old GOP, or poll + RTCP counters to manufacture a recovery controller. +3. Constrain each sender independently. A shared master must not lower a healthy + sibling or bypass the constrained connection's policy. Parameter-ceiling + checks are not network-shaping evidence. +4. Check the standard transform's frame ownership rules independently of the + legacy API. A permissive legacy write is not a portable encoder-factory API. + +Only pursue product integration if those cases preserve truthful diagnostics, +normal recovery and independent adaptation at a small maintenance cost. A +negative result is a valid conclusion; retain the ordinary Browser sender path +instead of adding an application BWE controller or sacrificing downstream +quality for an encode-count claim. + +### September 9 Results And Decision + +Nine isolated Chrome 152 scenarios extend the same loopback probe; the +[selected counters](./data/browser-encoded-fanout-continuation.json) contain no +media, SDP or ICE addresses. These are sender-parameter comparisons, not +effective network shaping or whole-process CPU measurements. + +The late-join defect is fixable without periodic work. After B becomes +connected, call A's existing +`setParameters(getParameters(), {encodingOptions: [{keyFrame: true}]})` once +and admit B at the next fresh keyframe. VP8/H264 began decoding after about +17/43 ms; each B decoded 210 frames, while A encoded 219 and B encoded zero. +The [pinned sender implementation](https://raw.githubusercontent.com/chromium/chromium/152.0.7977.0/third_party/blink/renderer/modules/peerconnection/rtc_rtp_sender.cc) +implements that option without a feature flag. This corrects the earlier +probe's missing recovery request; it does not connect future B feedback to A. + +| Comparison | A result | B result | +| --- | --- | --- | +| Shared VP8, A capped at 50 kbps | 320x180 | 320x180, although B is uncapped | +| Ordinary VP8, same A cap | 320x180 | 640x360 | +| Shared H264, A capped at 50 kbps | 480x270 | 480x270, although B is uncapped | +| Ordinary H264, same A cap | 480x270 | 640x360 | +| Shared VP8, only B capped at 50 kbps | 640x360 | 312,067 payload bytes in seven seconds, about 357 kbps | +| Standard transform, A original and B constructor-copy | A decodes 207 frames | B sends zero bytes and decodes nothing | +| Standard transform, constructor-copy also written to A | A encodes 210 frames but sends zero bytes | No delivery | + +The standard-transform cases completed without JavaScript errors. The tested +`new RTCEncodedVideoFrame(frame)` constructor is not a re-ownership API; +foreign-owner frames are discarded under the +[standard stream-processing contract](https://www.w3.org/TR/webrtc-encoded-transform/#stream-processing). +This does not claim every cloning operation has identical ownership behavior. + +Legacy B reports zero encoded/sent frames even while payload bytes increase. +A's receiver observes repeated RTCP sender reports; B's has no corresponding +remote-outbound report in these zero-encoder runs. The pinned native +[RTP sender](https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/call/rtp_video_sender.cc) +updates frame/timing accounting before the +[transform delegate's injection point](https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/modules/rtp_rtcp/source/rtp_sender_video_frame_transformer_delegate.cc). +Borrowing below that point bypasses those updates. This exposes an unverified +audio/video-clock boundary, not a measured audible desynchronization claim. + +**Decision: no product integration of legacy fanout.** The saved encode is real, +but the shared master's limitation damages a healthy sibling and the borrowed +connection's ordinary encoder policy no longer controls its payload. Adding a +live dummy encoder, independent budget polling, master election or timestamp +projection would introduce new ownership and recovery work rather than preserve +the current native adaptation contract. Standard Transform does not provide a +portable alternative injection point. Keep normal Browser senders and the +verified optional Browser-to-Client encoded fanout. + +A relevant future input API is Chromium's experimental +[`RTCEncodedSource` implementation](https://chromium-review.googlesource.com/c/chromium/src/+/8097283), +merged on 2026-09-07. It introduces encoded injection with keyframe and bitrate +feedback, but is absent from the checked M152 source. Do not ship a flag-dependent +path or assume this also supplies an automatic encoder/adaptation pool. Reopen +only when a supported browser exposes a documented input/feedback contract and +the same independent-quality, recovery, timing and resource comparisons pass. + +Reproduce the continuation with `--late-join --request-keyframe`, +`--limit-primary --request-keyframe`, `--limit-primary --ordinary`, +`--limit-secondary --request-keyframe`, `--standard`, or +`--standard --clone-own`; add `--h264` for codec comparisons. + ## Local Reconciliation Evidence Narada, Overcast and NICE show that overlay measurements can improve tree shape; diff --git a/docs/research/data/browser-encoded-fanout-continuation.json b/docs/research/data/browser-encoded-fanout-continuation.json new file mode 100644 index 0000000..37b9c68 --- /dev/null +++ b/docs/research/data/browser-encoded-fanout-continuation.json @@ -0,0 +1,368 @@ +{ + "date": "2026-09-09", + "browser": "Installed Chrome 152, isolated headless profile", + "scope": "Synthetic 640x360 canvas, loopback; sender parameter limits, not network shaping", + "productIntegrated": false, + "runs": [ + { + "name": "vp8-late-key", + "codec": "VP8", + "durationMs": 7000, + "primaryMaxBitrate": null, + "secondaryMaxBitrate": null, + "firstDecodedMs": 17.399999998509884, + "primaryKeys": 2, + "errors": [], + "aSend": { + "framesEncoded": 219, + "framesSent": 219, + "bytesSent": 361447, + "encodeSeconds": 0.162, + "reason": "none" + }, + "bSend": { + "framesEncoded": 0, + "framesSent": 0, + "bytesSent": 354993, + "encodeSeconds": 0, + "reason": "none" + }, + "aReceive": { + "framesDecoded": 219, + "reportsSent": 7 + }, + "bReceive": { + "framesDecoded": 210, + "reportsSent": 0 + }, + "aVideo": [ + 640, + 360 + ], + "bVideo": [ + 640, + 360 + ] + }, + { + "name": "h264-late-key", + "codec": "H264", + "durationMs": 7000, + "primaryMaxBitrate": null, + "secondaryMaxBitrate": null, + "firstDecodedMs": 43.399999998509884, + "primaryKeys": 2, + "errors": [], + "aSend": { + "framesEncoded": 219, + "framesSent": 219, + "bytesSent": 183570, + "encodeSeconds": 0.616, + "reason": "none" + }, + "bSend": { + "framesEncoded": 0, + "framesSent": 0, + "bytesSent": 179063, + "encodeSeconds": 0, + "reason": "none" + }, + "aReceive": { + "framesDecoded": 219, + "reportsSent": 7 + }, + "bReceive": { + "framesDecoded": 210, + "reportsSent": 0 + }, + "aVideo": [ + 640, + 360 + ], + "bVideo": [ + 640, + 360 + ] + }, + { + "name": "vp8-primary-limited-key", + "codec": "VP8", + "durationMs": 14000, + "primaryMaxBitrate": 50000, + "secondaryMaxBitrate": null, + "firstDecodedMs": 81.79999999701977, + "primaryKeys": 1, + "errors": [], + "aSend": { + "framesEncoded": 417, + "framesSent": 417, + "bytesSent": 86543, + "encodeSeconds": 0.177, + "reason": "bandwidth" + }, + "bSend": { + "framesEncoded": 0, + "framesSent": 0, + "bytesSent": 86543, + "encodeSeconds": 0, + "reason": "none" + }, + "aReceive": { + "framesDecoded": 417, + "reportsSent": 15 + }, + "bReceive": { + "framesDecoded": 417, + "reportsSent": 0 + }, + "aVideo": [ + 320, + 180 + ], + "bVideo": [ + 320, + 180 + ] + }, + { + "name": "vp8-primary-limited-ordinary", + "codec": "VP8", + "durationMs": 14000, + "primaryMaxBitrate": 50000, + "secondaryMaxBitrate": null, + "firstDecodedMs": null, + "primaryKeys": 1, + "errors": [], + "aSend": { + "framesEncoded": 415, + "framesSent": 415, + "bytesSent": 85701, + "encodeSeconds": 0.201, + "reason": "bandwidth" + }, + "bSend": { + "framesEncoded": 422, + "framesSent": 422, + "bytesSent": 654848, + "encodeSeconds": 0.396, + "reason": "none" + }, + "aReceive": { + "framesDecoded": 415, + "reportsSent": 15 + }, + "bReceive": { + "framesDecoded": 422, + "reportsSent": 14 + }, + "aVideo": [ + 320, + 180 + ], + "bVideo": [ + 640, + 360 + ] + }, + { + "name": "h264-primary-limited-key", + "codec": "H264", + "durationMs": 14000, + "primaryMaxBitrate": 50000, + "secondaryMaxBitrate": null, + "firstDecodedMs": 173.40000000223517, + "primaryKeys": 2, + "errors": [], + "aSend": { + "framesEncoded": 416, + "framesSent": 416, + "bytesSent": 80614, + "encodeSeconds": 0.25, + "reason": "bandwidth" + }, + "bSend": { + "framesEncoded": 0, + "framesSent": 0, + "bytesSent": 80614, + "encodeSeconds": 0, + "reason": "none" + }, + "aReceive": { + "framesDecoded": 416, + "reportsSent": 15 + }, + "bReceive": { + "framesDecoded": 416, + "reportsSent": 0 + }, + "aVideo": [ + 480, + 270 + ], + "bVideo": [ + 480, + 270 + ] + }, + { + "name": "h264-primary-limited-ordinary", + "codec": "H264", + "durationMs": 14000, + "primaryMaxBitrate": 50000, + "secondaryMaxBitrate": null, + "firstDecodedMs": 181.70000000298023, + "primaryKeys": 2, + "errors": [], + "aSend": { + "framesEncoded": 415, + "framesSent": 415, + "bytesSent": 80241, + "encodeSeconds": 0.269, + "reason": "bandwidth" + }, + "bSend": { + "framesEncoded": 416, + "framesSent": 416, + "bytesSent": 294216, + "encodeSeconds": 1.185, + "reason": "none" + }, + "aReceive": { + "framesDecoded": 415, + "reportsSent": 15 + }, + "bReceive": { + "framesDecoded": 416, + "reportsSent": 14 + }, + "aVideo": [ + 480, + 270 + ], + "bVideo": [ + 640, + 360 + ] + }, + { + "name": "vp8-limited-key", + "codec": "VP8", + "durationMs": 7000, + "primaryMaxBitrate": null, + "secondaryMaxBitrate": 50000, + "firstDecodedMs": 25.900000002235174, + "primaryKeys": 1, + "errors": [], + "aSend": { + "framesEncoded": 208, + "framesSent": 208, + "bytesSent": 389433, + "encodeSeconds": 0.176, + "reason": "none" + }, + "bSend": { + "framesEncoded": 0, + "framesSent": 0, + "bytesSent": 312067, + "encodeSeconds": 0, + "reason": "none" + }, + "aReceive": { + "framesDecoded": 208, + "reportsSent": 8 + }, + "bReceive": { + "framesDecoded": 158, + "reportsSent": 0 + }, + "aVideo": [ + 640, + 360 + ], + "bVideo": [ + 640, + 360 + ] + }, + { + "name": "vp8-standard", + "codec": "VP8", + "durationMs": 7000, + "primaryMaxBitrate": null, + "secondaryMaxBitrate": null, + "firstDecodedMs": null, + "primaryKeys": 0, + "errors": [], + "aSend": { + "framesEncoded": 208, + "framesSent": 208, + "bytesSent": 386466, + "encodeSeconds": 0.168, + "reason": "none" + }, + "bSend": { + "framesEncoded": 0, + "framesSent": 0, + "bytesSent": 0, + "encodeSeconds": 0, + "reason": "none" + }, + "aReceive": { + "framesDecoded": 207, + "reportsSent": 8 + }, + "bReceive": { + "framesDecoded": 0, + "reportsSent": 0 + }, + "aVideo": [ + 640, + 360 + ], + "bVideo": [ + 0, + 0 + ] + }, + { + "name": "vp8-standard-clone-own", + "codec": "VP8", + "durationMs": 7000, + "primaryMaxBitrate": null, + "secondaryMaxBitrate": null, + "firstDecodedMs": null, + "primaryKeys": 0, + "errors": [], + "aSend": { + "framesEncoded": 210, + "framesSent": 210, + "bytesSent": 0, + "encodeSeconds": 0.187, + "reason": "none" + }, + "bSend": { + "framesEncoded": 0, + "framesSent": 0, + "bytesSent": 0, + "encodeSeconds": 0, + "reason": "none" + }, + "aReceive": { + "framesDecoded": 0, + "reportsSent": 0 + }, + "bReceive": { + "framesDecoded": 0, + "reportsSent": 0 + }, + "aVideo": [ + 0, + 0 + ], + "bVideo": [ + 0, + 0 + ] + } + ] +} diff --git a/docs/research/data/browser-encoded-fanout.json b/docs/research/data/browser-encoded-fanout.json new file mode 100644 index 0000000..9812d67 --- /dev/null +++ b/docs/research/data/browser-encoded-fanout.json @@ -0,0 +1,287 @@ +{ + "date": "2026-09-08", + "scope": "Chrome152 legacy createEncodedStreams, synthetic canvas, two loopback WebRTC connections, no Client or SFU; no loss/bandwidth shaping", + "standardApi": false, + "productIntegrated": false, + "sourceSha256": "59015dacabf258ffa6f266c32815e9a20fb1e3d90487a0f0267f153db555145b", + "runs": [ + { + "codec": "VP8", + "lateJoin": false, + "browser": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/152.0.0.0 Safari/537.36", + "sharedFrames": 208, + "dummyFrames": 0, + "errors": [], + "aSend": [ + { + "type": "outbound-rtp", + "framesEncoded": 208, + "framesSent": 208, + "frameWidth": 640, + "frameHeight": 360, + "framesPerSecond": 30, + "bytesSent": 390630, + "totalEncodeTime": 0.175, + "qualityLimitationReason": "none", + "targetBitrate": 1700000, + "pliCount": 0, + "nackCount": 0 + } + ], + "bSend": [ + { + "type": "outbound-rtp", + "framesEncoded": 0, + "framesSent": 0, + "bytesSent": 390630, + "totalEncodeTime": 0, + "qualityLimitationReason": "none", + "pliCount": 0, + "nackCount": 0 + } + ], + "aReceive": [ + { + "type": "inbound-rtp", + "framesDecoded": 208, + "framesReceived": 208, + "frameWidth": 640, + "frameHeight": 360, + "framesPerSecond": 30, + "bytesReceived": 390630, + "pliCount": 0, + "nackCount": 0 + } + ], + "bReceive": [ + { + "type": "inbound-rtp", + "framesDecoded": 208, + "framesReceived": 208, + "frameWidth": 640, + "frameHeight": 360, + "framesPerSecond": 30, + "bytesReceived": 390630, + "pliCount": 0, + "nackCount": 0 + } + ], + "aVideo": [ + 640, + 360 + ], + "bVideo": [ + 640, + 360 + ] + }, + { + "codec": "H264", + "lateJoin": false, + "browser": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/152.0.0.0 Safari/537.36", + "sharedFrames": 206, + "dummyFrames": 0, + "errors": [], + "aSend": [ + { + "type": "outbound-rtp", + "framesEncoded": 206, + "framesSent": 206, + "frameWidth": 640, + "frameHeight": 360, + "framesPerSecond": 30, + "bytesSent": 175079, + "totalEncodeTime": 0.284, + "qualityLimitationReason": "none", + "targetBitrate": 1700000, + "pliCount": 0, + "nackCount": 0 + } + ], + "bSend": [ + { + "type": "outbound-rtp", + "framesEncoded": 0, + "framesSent": 0, + "bytesSent": 175079, + "totalEncodeTime": 0, + "qualityLimitationReason": "none", + "pliCount": 0, + "nackCount": 0 + } + ], + "aReceive": [ + { + "type": "inbound-rtp", + "framesDecoded": 206, + "framesReceived": 206, + "frameWidth": 640, + "frameHeight": 360, + "framesPerSecond": 30, + "bytesReceived": 175079, + "pliCount": 0, + "nackCount": 0 + } + ], + "bReceive": [ + { + "type": "inbound-rtp", + "framesDecoded": 206, + "framesReceived": 206, + "frameWidth": 640, + "frameHeight": 360, + "framesPerSecond": 30, + "bytesReceived": 175079, + "pliCount": 0, + "nackCount": 0 + } + ], + "aVideo": [ + 640, + 360 + ], + "bVideo": [ + 640, + 360 + ] + }, + { + "codec": "VP8", + "lateJoin": true, + "browser": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/152.0.0.0 Safari/537.36", + "sharedFrames": 220, + "dummyFrames": 0, + "errors": [], + "aSend": [ + { + "type": "outbound-rtp", + "framesEncoded": 220, + "framesSent": 220, + "frameWidth": 640, + "frameHeight": 360, + "framesPerSecond": 30, + "bytesSent": 408158, + "totalEncodeTime": 0.173, + "qualityLimitationReason": "none", + "targetBitrate": 1700000, + "pliCount": 0, + "nackCount": 0 + } + ], + "bSend": [ + { + "type": "outbound-rtp", + "framesEncoded": 0, + "framesSent": 0, + "bytesSent": 400636, + "totalEncodeTime": 0, + "qualityLimitationReason": "none", + "pliCount": 34, + "nackCount": 0 + } + ], + "aReceive": [ + { + "type": "inbound-rtp", + "framesDecoded": 220, + "framesReceived": 220, + "frameWidth": 640, + "frameHeight": 360, + "framesPerSecond": 30, + "bytesReceived": 408158, + "pliCount": 0, + "nackCount": 0 + } + ], + "bReceive": [ + { + "type": "inbound-rtp", + "framesDecoded": 0, + "framesReceived": 0, + "bytesReceived": 400636, + "pliCount": 34, + "nackCount": 0 + } + ], + "aVideo": [ + 640, + 360 + ], + "bVideo": [ + 0, + 0 + ] + }, + { + "codec": "VP8", + "lateJoin": false, + "secondaryMaxBitrate": 50000, + "browser": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/152.0.0.0 Safari/537.36", + "sharedFrames": 208, + "dummyFrames": 0, + "errors": [], + "aSend": [ + { + "type": "outbound-rtp", + "framesEncoded": 208, + "framesSent": 208, + "frameWidth": 640, + "frameHeight": 360, + "framesPerSecond": 30, + "bytesSent": 387287, + "totalEncodeTime": 0.179, + "qualityLimitationReason": "none", + "targetBitrate": 1700000, + "pliCount": 0, + "nackCount": 0 + } + ], + "bSend": [ + { + "type": "outbound-rtp", + "framesEncoded": 0, + "framesSent": 0, + "bytesSent": 387287, + "totalEncodeTime": 0, + "qualityLimitationReason": "none", + "pliCount": 0, + "nackCount": 0 + } + ], + "aReceive": [ + { + "type": "inbound-rtp", + "framesDecoded": 207, + "framesReceived": 208, + "frameWidth": 640, + "frameHeight": 360, + "framesPerSecond": 30, + "bytesReceived": 387287, + "pliCount": 0, + "nackCount": 0 + } + ], + "bReceive": [ + { + "type": "inbound-rtp", + "framesDecoded": 207, + "framesReceived": 208, + "frameWidth": 640, + "frameHeight": 360, + "framesPerSecond": 30, + "bytesReceived": 387287, + "pliCount": 0, + "nackCount": 0 + } + ], + "aVideo": [ + 640, + 360 + ], + "bVideo": [ + 640, + 360 + ] + } + ] +} diff --git a/docs/research/data/node-local-browser.json b/docs/research/data/node-local-browser.json new file mode 100644 index 0000000..390fb17 --- /dev/null +++ b/docs/research/data/node-local-browser.json @@ -0,0 +1 @@ +{"completed":true,"scope":"WebCodecs software preference; independent child decode; no RTP, network, hardware or perceptual acceptance","userAgent":"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/152.0.0.0 Safari/537.36","support":{"vp8":true,"avc1.420028":true},"profiles":[[480,270,312500],[960,540,1250000],[1920,1080,5000000]],"fps":30,"phaseFrames":32,"trace":[[2,2],[2,0],[0,0],[1,1],[2,2],[2,0],[2,2]],"results":[{"round":1,"codec":"vp8","policy":"exact","role":"host","encoded":[96,32,160],"decoded":[224,224],"ingressDecodes":0,"creates":5,"generatedBytes":[122548,136335,3328684],"selectedBytes":[3512141,2046547],"switches":[{"index":32,"child":1,"layer":0,"ms":5.399999976158142,"bytes":2510},{"index":64,"child":0,"layer":0,"ms":6.5,"bytes":4426},{"index":96,"child":0,"layer":1,"ms":9,"bytes":8218},{"index":96,"child":1,"layer":1,"ms":9,"bytes":8218},{"index":128,"child":0,"layer":2,"ms":22.200000047683716,"bytes":21119},{"index":128,"child":1,"layer":2,"ms":22.200000047683716,"bytes":21119},{"index":160,"child":1,"layer":0,"ms":5.5,"bytes":2510},{"index":192,"child":1,"layer":2,"ms":15.399999976158142,"bytes":24215}],"elapsedMs":7467.600000023842,"metrics":{"management":{"mean":0.010267858526536397,"p95":0.10000002384185791,"max":0.20000004768371582},"input":{"mean":0.1477678609745843,"p95":0.20000004768371582,"max":0.30000007152557373},"encodeReady":{"mean":2.772767852459635,"p95":5.400000095367432,"max":14.100000023841858},"allReceivers":{"mean":6.7459821456245015,"p95":15.100000023841858,"max":43.10000002384186},"processing":{"mean":6.747767860335963,"p95":15.100000023841858,"max":43.10000002384186}},"overFrameBudget":1,"phases":[{"demand":[2,2],"frames":32,"work":{"mean":9.996875006705523,"p95":21,"max":22.399999976158142}},{"demand":[2,0],"frames":32,"work":{"mean":6.725000001490116,"p95":13,"max":18.100000023841858}},{"demand":[0,0],"frames":32,"work":{"mean":1.5218749940395355,"p95":3.5,"max":6.5}},{"demand":[1,1],"frames":32,"work":{"mean":3.328125,"p95":7.199999928474426,"max":9}},{"demand":[2,2],"frames":32,"work":{"mean":9.681250013411045,"p95":20.600000023841858,"max":22.200000047683716}},{"demand":[2,0],"frames":32,"work":{"mean":6.687500007450581,"p95":13.100000023841858,"max":18.299999952316284}},{"demand":[2,2],"frames":32,"work":{"mean":9.293749999254942,"p95":16.799999952316284,"max":43.10000002384186}}]},{"round":1,"codec":"vp8","policy":"envelope","role":"host","encoded":[224,192,160],"decoded":[224,224],"ingressDecodes":0,"creates":5,"generatedBytes":[283892,891748,3328684],"selectedBytes":[3510224,2055606],"switches":[{"index":32,"child":1,"layer":0,"ms":4.299999952316284,"bytes":4426},{"index":64,"child":0,"layer":0,"ms":7.400000095367432,"bytes":4391},{"index":96,"child":0,"layer":1,"ms":8.299999952316284,"bytes":8218},{"index":96,"child":1,"layer":1,"ms":8.299999952316284,"bytes":8218},{"index":128,"child":0,"layer":2,"ms":20.899999976158142,"bytes":21119},{"index":128,"child":1,"layer":2,"ms":20.899999976158142,"bytes":21119},{"index":160,"child":1,"layer":0,"ms":4.399999976158142,"bytes":6076},{"index":192,"child":1,"layer":2,"ms":15.5,"bytes":24215}],"elapsedMs":7466.700000047684,"metrics":{"management":{"mean":0.015178571854318892,"p95":0.10000002384185791,"max":0.20000004768371582},"input":{"mean":0.1455357117312295,"p95":0.20000004768371582,"max":0.30000007152557373},"encodeReady":{"mean":2.8874999987227574,"p95":5.700000047683716,"max":14.600000023841858},"allReceivers":{"mean":6.89062499627471,"p95":15.300000071525574,"max":42.39999997615814},"processing":{"mean":6.892410710986171,"p95":15.300000071525574,"max":42.39999997615814}},"overFrameBudget":1,"phases":[{"demand":[2,2],"frames":32,"work":{"mean":10.118750002235174,"p95":20.899999976158142,"max":24.40000009536743}},{"demand":[2,0],"frames":32,"work":{"mean":6.846874989569187,"p95":13.199999928474426,"max":18.600000023841858}},{"demand":[0,0],"frames":32,"work":{"mean":1.5406250059604645,"p95":3.899999976158142,"max":7.400000095367432}},{"demand":[1,1],"frames":32,"work":{"mean":3.393749989569187,"p95":8.299999952316284,"max":8.399999976158142}},{"demand":[2,2],"frames":32,"work":{"mean":10.015624985098839,"p95":20.899999976158142,"max":21}},{"demand":[2,0],"frames":32,"work":{"mean":6.887500010430813,"p95":13,"max":18.399999976158142}},{"demand":[2,2],"frames":32,"work":{"mean":9.443749994039536,"p95":17,"max":42.39999997615814}}]},{"round":1,"codec":"vp8","policy":"exact","role":"relay","encoded":[96,32,0],"decoded":[224,224],"ingressDecodes":128,"creates":3,"generatedBytes":[120010,140534,0],"selectedBytes":[2999436,1947552],"switches":[{"index":32,"child":1,"layer":0,"ms":14.599999904632568,"bytes":2510},{"index":64,"child":0,"layer":0,"ms":7.899999976158142,"bytes":4299},{"index":96,"child":0,"layer":1,"ms":12.299999952316284,"bytes":6098},{"index":96,"child":1,"layer":1,"ms":12.299999952316284,"bytes":6098},{"index":128,"child":0,"layer":2,"ms":11.600000023841858,"bytes":21119},{"index":128,"child":1,"layer":2,"ms":11.600000023841858,"bytes":21119},{"index":160,"child":1,"layer":0,"ms":15.400000095367432,"bytes":2510},{"index":192,"child":1,"layer":2,"ms":9.100000023841858,"bytes":21119}],"elapsedMs":7467.100000023842,"metrics":{"management":{"mean":0.008482142218521662,"p95":0.10000002384185791,"max":0.19999992847442627},"input":{"mean":2.7388392851821015,"p95":9.199999928474426,"max":14.299999952316284},"encodeReady":{"mean":0.49910714477300644,"p95":1.5,"max":3.299999952316284},"allReceivers":{"mean":6.433035716414452,"p95":12.399999976158142,"max":18.100000023841858},"processing":{"mean":6.43526785873941,"p95":12.399999976158142,"max":18.100000023841858}},"overFrameBudget":0,"phases":[{"demand":[2,2],"frames":32,"work":{"mean":6.665624991059303,"p95":14.399999976158142,"max":18.100000023841858}},{"demand":[2,0],"frames":32,"work":{"mean":7.403124991804361,"p95":14.599999904632568,"max":15.299999952316284}},{"demand":[0,0],"frames":32,"work":{"mean":4.4343750067055225,"p95":7.899999976158142,"max":8.200000047683716}},{"demand":[1,1],"frames":32,"work":{"mean":6.290624987334013,"p95":12.299999952316284,"max":13.100000023841858}},{"demand":[2,2],"frames":32,"work":{"mean":6.350000020116568,"p95":11.600000023841858,"max":14.299999952316284}},{"demand":[2,0],"frames":32,"work":{"mean":7.40625000372529,"p95":15.399999976158142,"max":15.400000095367432}},{"demand":[2,2],"frames":32,"work":{"mean":6.496875010430813,"p95":13.800000071525574,"max":14.299999952316284}}]},{"round":1,"codec":"vp8","policy":"envelope","role":"relay","encoded":[128,96,0],"decoded":[224,224],"ingressDecodes":128,"creates":5,"generatedBytes":[162195,421602,0],"selectedBytes":[2999436,1947552],"switches":[{"index":32,"child":1,"layer":0,"ms":15.100000023841858,"bytes":2510},{"index":64,"child":0,"layer":0,"ms":8.300000071525574,"bytes":4299},{"index":96,"child":0,"layer":1,"ms":10.300000071525574,"bytes":6098},{"index":96,"child":1,"layer":1,"ms":10.300000071525574,"bytes":6098},{"index":128,"child":0,"layer":2,"ms":11.399999976158142,"bytes":21119},{"index":128,"child":1,"layer":2,"ms":11.399999976158142,"bytes":21119},{"index":160,"child":1,"layer":0,"ms":12.5,"bytes":2510},{"index":192,"child":1,"layer":2,"ms":9.300000071525574,"bytes":21119}],"elapsedMs":7467,"metrics":{"management":{"mean":0.013392857675041472,"p95":0.10000002384185791,"max":0.20000004768371582},"input":{"mean":2.7053571471146176,"p95":8.399999976158142,"max":14.599999904632568},"encodeReady":{"mean":0.697321431445224,"p95":1.7999999523162842,"max":3.5},"allReceivers":{"mean":6.357142864061253,"p95":11.399999976158142,"max":15.699999928474426},"processing":{"mean":6.446875008089202,"p95":11.5,"max":17}},"overFrameBudget":0,"phases":[{"demand":[2,2],"frames":32,"work":{"mean":6.4625000059604645,"p95":14.399999976158142,"max":14.800000071525574}},{"demand":[2,0],"frames":32,"work":{"mean":7.706249997019768,"p95":15.100000023841858,"max":16.600000023841858}},{"demand":[0,0],"frames":32,"work":{"mean":4.4687500186264515,"p95":8.200000047683716,"max":8.300000071525574}},{"demand":[1,1],"frames":32,"work":{"mean":6.312500014901161,"p95":10.300000071525574,"max":12.899999976158142}},{"demand":[2,2],"frames":32,"work":{"mean":6.284375004470348,"p95":11.399999976158142,"max":14.299999952316284}},{"demand":[2,0],"frames":32,"work":{"mean":7.6343750059604645,"p95":12.600000023841858,"max":17}},{"demand":[2,2],"frames":32,"work":{"mean":6.259375009685755,"p95":11.200000047683716,"max":14.300000071525574}}]},{"round":2,"codec":"vp8","policy":"envelope","role":"host","encoded":[224,192,160],"decoded":[224,224],"ingressDecodes":0,"creates":5,"generatedBytes":[283892,891748,3328684],"selectedBytes":[3510224,2055606],"switches":[{"index":32,"child":1,"layer":0,"ms":4.200000047683716,"bytes":4426},{"index":64,"child":0,"layer":0,"ms":6.699999928474426,"bytes":4391},{"index":96,"child":0,"layer":1,"ms":8.5,"bytes":8218},{"index":96,"child":1,"layer":1,"ms":8.5,"bytes":8218},{"index":128,"child":0,"layer":2,"ms":21.100000023841858,"bytes":21119},{"index":128,"child":1,"layer":2,"ms":21.100000023841858,"bytes":21119},{"index":160,"child":1,"layer":0,"ms":4.199999928474426,"bytes":6076},{"index":192,"child":1,"layer":2,"ms":15.400000095367432,"bytes":24215}],"elapsedMs":7467.100000023842,"metrics":{"management":{"mean":0.011160713753529958,"p95":0.10000002384185791,"max":0.19999992847442627},"input":{"mean":0.14241070885743415,"p95":0.20000004768371582,"max":0.30000007152557373},"encodeReady":{"mean":2.9459821454116275,"p95":5.800000071525574,"max":14.700000047683716},"allReceivers":{"mean":6.968749995210341,"p95":15.200000047683716,"max":42.300000071525574},"processing":{"mean":6.971428567277534,"p95":15.200000047683716,"max":42.300000071525574}},"overFrameBudget":1,"phases":[{"demand":[2,2],"frames":32,"work":{"mean":10.274999994784594,"p95":21,"max":21.699999928474426}},{"demand":[2,0],"frames":32,"work":{"mean":7.0031249932944775,"p95":13.100000023841858,"max":22.299999952316284}},{"demand":[0,0],"frames":32,"work":{"mean":1.449999999254942,"p95":3.5,"max":6.799999952316284}},{"demand":[1,1],"frames":32,"work":{"mean":3.3281250074505806,"p95":7,"max":8.5}},{"demand":[2,2],"frames":32,"work":{"mean":10.240624986588955,"p95":21.100000023841858,"max":25}},{"demand":[2,0],"frames":32,"work":{"mean":6.974999990314245,"p95":13.199999928474426,"max":21.800000071525574}},{"demand":[2,2],"frames":32,"work":{"mean":9.528124999254942,"p95":17.100000023841858,"max":42.300000071525574}}]},{"round":2,"codec":"vp8","policy":"exact","role":"host","encoded":[96,32,160],"decoded":[224,224],"ingressDecodes":0,"creates":5,"generatedBytes":[122548,136335,3328684],"selectedBytes":[3512141,2046547],"switches":[{"index":32,"child":1,"layer":0,"ms":4.899999976158142,"bytes":2510},{"index":64,"child":0,"layer":0,"ms":8.299999952316284,"bytes":4426},{"index":96,"child":0,"layer":1,"ms":9.200000047683716,"bytes":8218},{"index":96,"child":1,"layer":1,"ms":9.200000047683716,"bytes":8218},{"index":128,"child":0,"layer":2,"ms":21.399999976158142,"bytes":21119},{"index":128,"child":1,"layer":2,"ms":21.399999976158142,"bytes":21119},{"index":160,"child":1,"layer":0,"ms":6,"bytes":2510},{"index":192,"child":1,"layer":2,"ms":15.5,"bytes":24215}],"elapsedMs":7466.600000023842,"metrics":{"management":{"mean":0.008928571960755758,"p95":0.10000002384185791,"max":0.20000004768371582},"input":{"mean":0.15401785661067283,"p95":0.20000004768371582,"max":0.40000009536743164},"encodeReady":{"mean":2.882142853524004,"p95":6.399999976158142,"max":16.199999928474426},"allReceivers":{"mean":6.957589287310839,"p95":15.5,"max":41.800000071525574},"processing":{"mean":6.962053573025124,"p95":15.5,"max":41.800000071525574}},"overFrameBudget":1,"phases":[{"demand":[2,2],"frames":32,"work":{"mean":10.081250008195639,"p95":20.5,"max":26.5}},{"demand":[2,0],"frames":32,"work":{"mean":6.6906249932944775,"p95":15,"max":18.399999976158142}},{"demand":[0,0],"frames":32,"work":{"mean":1.5874999910593033,"p95":3.5,"max":8.299999952316284}},{"demand":[1,1],"frames":32,"work":{"mean":3.381249997764826,"p95":7.399999976158142,"max":9.200000047683716}},{"demand":[2,2],"frames":32,"work":{"mean":11.012499991804361,"p95":20.699999928474426,"max":21.399999976158142}},{"demand":[2,0],"frames":32,"work":{"mean":6.825000025331974,"p95":13.300000071525574,"max":18.299999952316284}},{"demand":[2,2],"frames":32,"work":{"mean":9.15625000372529,"p95":16.600000023841858,"max":41.800000071525574}}]},{"round":2,"codec":"vp8","policy":"envelope","role":"relay","encoded":[128,96,0],"decoded":[224,224],"ingressDecodes":128,"creates":5,"generatedBytes":[162195,421602,0],"selectedBytes":[2999436,1947552],"switches":[{"index":32,"child":1,"layer":0,"ms":15.199999928474426,"bytes":2510},{"index":64,"child":0,"layer":0,"ms":8.200000047683716,"bytes":4299},{"index":96,"child":0,"layer":1,"ms":10.899999976158142,"bytes":6098},{"index":96,"child":1,"layer":1,"ms":10.899999976158142,"bytes":6098},{"index":128,"child":0,"layer":2,"ms":12.5,"bytes":21119},{"index":128,"child":1,"layer":2,"ms":12.5,"bytes":21119},{"index":160,"child":1,"layer":0,"ms":11.400000095367432,"bytes":2510},{"index":192,"child":1,"layer":2,"ms":9,"bytes":21119}],"elapsedMs":7467.200000047684,"metrics":{"management":{"mean":0.014285713966403688,"p95":0.10000002384185791,"max":0.19999992847442627},"input":{"mean":2.721875005002533,"p95":9,"max":14.200000047683716},"encodeReady":{"mean":0.6950892859271595,"p95":1.7999999523162842,"max":3.5},"allReceivers":{"mean":6.492410720459053,"p95":11.400000095367432,"max":17.899999976158142},"processing":{"mean":6.583928579198463,"p95":11.700000047683716,"max":17.899999976158142}},"overFrameBudget":0,"phases":[{"demand":[2,2],"frames":32,"work":{"mean":6.565625000745058,"p95":14.800000071525574,"max":17.899999976158142}},{"demand":[2,0],"frames":32,"work":{"mean":7.659375008195639,"p95":15.199999928474426,"max":16.5}},{"demand":[0,0],"frames":32,"work":{"mean":4.653125029057264,"p95":8.700000047683716,"max":8.899999976158142}},{"demand":[1,1],"frames":32,"work":{"mean":6.303125001490116,"p95":10.899999976158142,"max":12.799999952316284}},{"demand":[2,2],"frames":32,"work":{"mean":6.737499997019768,"p95":12.5,"max":14.899999976158142}},{"demand":[2,0],"frames":32,"work":{"mean":7.709375005215406,"p95":12.300000071525574,"max":16.700000047683716}},{"demand":[2,2],"frames":32,"work":{"mean":6.459375012665987,"p95":10.899999976158142,"max":14.700000047683716}}]},{"round":2,"codec":"vp8","policy":"exact","role":"relay","encoded":[96,32,0],"decoded":[224,224],"ingressDecodes":128,"creates":3,"generatedBytes":[120010,140534,0],"selectedBytes":[2999436,1947552],"switches":[{"index":32,"child":1,"layer":0,"ms":14.700000047683716,"bytes":2510},{"index":64,"child":0,"layer":0,"ms":8,"bytes":4299},{"index":96,"child":0,"layer":1,"ms":11.800000071525574,"bytes":6098},{"index":96,"child":1,"layer":1,"ms":11.800000071525574,"bytes":6098},{"index":128,"child":0,"layer":2,"ms":12.199999928474426,"bytes":21119},{"index":128,"child":1,"layer":2,"ms":12.199999928474426,"bytes":21119},{"index":160,"child":1,"layer":0,"ms":11.5,"bytes":2510},{"index":192,"child":1,"layer":2,"ms":9.299999952316284,"bytes":21119}],"elapsedMs":7467.5,"metrics":{"management":{"mean":0.008482142750705992,"p95":0.10000002384185791,"max":0.10000002384185791},"input":{"mean":2.6950892885880813,"p95":8.5,"max":14.400000095367432},"encodeReady":{"mean":0.49330357355730875,"p95":1.4000000953674316,"max":3.200000047683716},"allReceivers":{"mean":6.303571429635797,"p95":11.5,"max":15.5},"processing":{"mean":6.305803573025124,"p95":11.5,"max":15.5}},"overFrameBudget":0,"phases":[{"demand":[2,2],"frames":32,"work":{"mean":6.362500011920929,"p95":14.200000047683716,"max":14.899999976158142}},{"demand":[2,0],"frames":32,"work":{"mean":7.318750008940697,"p95":14.700000047683716,"max":15.5}},{"demand":[0,0],"frames":32,"work":{"mean":4.449999995529652,"p95":8,"max":8.199999928474426}},{"demand":[1,1],"frames":32,"work":{"mean":6.212500009685755,"p95":11.800000071525574,"max":13}},{"demand":[2,2],"frames":32,"work":{"mean":6.284374982118607,"p95":12.199999928474426,"max":14.199999928474426}},{"demand":[2,0],"frames":32,"work":{"mean":7.159375008195639,"p95":11.5,"max":15.199999928474426}},{"demand":[2,2],"frames":32,"work":{"mean":6.353124994784594,"p95":10.600000023841858,"max":14.399999976158142}}]},{"round":1,"codec":"avc1.420028","policy":"exact","role":"host","encoded":[96,32,160],"decoded":[224,224],"ingressDecodes":0,"creates":5,"generatedBytes":[185868,168672,3257425],"selectedBytes":[3477507,2312313],"switches":[{"index":32,"child":1,"layer":0,"ms":3.1999999284744263,"bytes":32563},{"index":64,"child":0,"layer":0,"ms":3.899999976158142,"bytes":17323},{"index":96,"child":0,"layer":1,"ms":99.40000009536743,"bytes":79574},{"index":96,"child":1,"layer":1,"ms":100.20000004768372,"bytes":79574},{"index":128,"child":0,"layer":2,"ms":193.10000002384186,"bytes":146641},{"index":128,"child":1,"layer":2,"ms":194.79999995231628,"bytes":146641},{"index":160,"child":1,"layer":0,"ms":3.100000023841858,"bytes":32563},{"index":192,"child":1,"layer":2,"ms":10.899999976158142,"bytes":99623}],"elapsedMs":7466.899999976158,"metrics":{"management":{"mean":0.00714285671710968,"p95":0.09999990463256836,"max":0.20000004768371582},"input":{"mean":0.14553570321628026,"p95":0.20000004768371582,"max":0.30000007152557373},"encodeReady":{"mean":4.532142860016653,"p95":5.600000023841858,"max":190.39999997615814},"allReceivers":{"mean":5.666964279753821,"p95":8,"max":194.79999995231628},"processing":{"mean":5.669196423143148,"p95":8,"max":194.79999995231628}},"overFrameBudget":3,"phases":[{"demand":[2,2],"frames":32,"work":{"mean":8.596874989569187,"p95":10.300000071525574,"max":128.69999992847443}},{"demand":[2,0],"frames":32,"work":{"mean":4.643749989569187,"p95":7.699999928474426,"max":9.200000047683716}},{"demand":[0,0],"frames":32,"work":{"mean":1.1031250096857548,"p95":2.3000000715255737,"max":3.899999976158142}},{"demand":[1,1],"frames":32,"work":{"mean":4.646874986588955,"p95":4.299999952316284,"max":100.20000004768372}},{"demand":[2,2],"frames":32,"work":{"mean":10.59375,"p95":10.799999952316284,"max":194.79999995231628}},{"demand":[2,0],"frames":32,"work":{"mean":4.6875,"p95":8,"max":9.200000047683716}},{"demand":[2,2],"frames":32,"work":{"mean":5.412499986588955,"p95":9.200000047683716,"max":10.899999976158142}}]},{"round":1,"codec":"avc1.420028","policy":"envelope","role":"host","encoded":[224,192,160],"decoded":[224,224],"ingressDecodes":0,"creates":5,"generatedBytes":[384469,1023753,3257425],"selectedBytes":[3477015,2280184],"switches":[{"index":32,"child":1,"layer":0,"ms":2.5,"bytes":17323},{"index":64,"child":0,"layer":0,"ms":2.799999952316284,"bytes":17323},{"index":96,"child":0,"layer":1,"ms":97.39999997615814,"bytes":79574},{"index":96,"child":1,"layer":1,"ms":98,"bytes":79574},{"index":128,"child":0,"layer":2,"ms":36,"bytes":146641},{"index":128,"child":1,"layer":2,"ms":37.5,"bytes":146641},{"index":160,"child":1,"layer":0,"ms":2.200000047683716,"bytes":17324},{"index":192,"child":1,"layer":2,"ms":13,"bytes":99623}],"elapsedMs":7467,"metrics":{"management":{"mean":0.008928571960755758,"p95":0.10000002384185791,"max":0.10000002384185791},"input":{"mean":0.15401785980377877,"p95":0.20000004768371582,"max":0.30000007152557373},"encodeReady":{"mean":4.3147321455180645,"p95":7.799999952316284,"max":96},"allReceivers":{"mean":5.444196431764534,"p95":10.399999976158142,"max":98},"processing":{"mean":5.449553574834551,"p95":10.399999976158142,"max":98}},"overFrameBudget":3,"phases":[{"demand":[2,2],"frames":32,"work":{"mean":6.531250011175871,"p95":10.399999976158142,"max":52.40000009536743}},{"demand":[2,0],"frames":32,"work":{"mean":4.8093750067055225,"p95":8.899999976158142,"max":9.299999952316284}},{"demand":[0,0],"frames":32,"work":{"mean":1.043749999254942,"p95":2.899999976158142,"max":3}},{"demand":[1,1],"frames":32,"work":{"mean":4.5875000059604645,"p95":4.699999928474426,"max":98}},{"demand":[2,2],"frames":32,"work":{"mean":7.378124974668026,"p95":13.299999952316284,"max":37.5}},{"demand":[2,0],"frames":32,"work":{"mean":6.443750012665987,"p95":10.600000023841858,"max":12.5}},{"demand":[2,2],"frames":32,"work":{"mean":7.353125013411045,"p95":13,"max":14.100000023841858}}]},{"round":1,"codec":"avc1.420028","policy":"exact","role":"relay","encoded":[96,32,0],"decoded":[224,224],"ingressDecodes":128,"creates":3,"generatedBytes":[185530,169918,0],"selectedBytes":[3510902,2330426],"switches":[{"index":32,"child":1,"layer":0,"ms":7,"bytes":25704},{"index":64,"child":0,"layer":0,"ms":4.200000047683716,"bytes":17134},{"index":96,"child":0,"layer":1,"ms":93.80000007152557,"bytes":56516},{"index":96,"child":1,"layer":1,"ms":94.30000007152557,"bytes":56516},{"index":128,"child":0,"layer":2,"ms":2.6999999284744263,"bytes":146641},{"index":128,"child":1,"layer":2,"ms":4.299999952316284,"bytes":146641},{"index":160,"child":1,"layer":0,"ms":4.799999952316284,"bytes":25704},{"index":192,"child":1,"layer":2,"ms":4.299999952316284,"bytes":146641}],"elapsedMs":7467.200000047684,"metrics":{"management":{"mean":0.008928570364202772,"p95":0.10000002384185791,"max":0.20000004768371582},"input":{"mean":0.4941964255911963,"p95":1.5999999046325684,"max":4.399999976158142},"encodeReady":{"mean":0.9441964312323502,"p95":1.3000000715255737,"max":90.70000004768372},"allReceivers":{"mean":2.3008928623582636,"p95":3.899999976158142,"max":94.30000007152557},"processing":{"mean":2.3017857197139944,"p95":3.899999976158142,"max":94.30000007152557}},"overFrameBudget":1,"phases":[{"demand":[2,2],"frames":32,"work":{"mean":1.818750023841858,"p95":2.899999976158142,"max":11.200000047683716}},{"demand":[2,0],"frames":32,"work":{"mean":2.1000000089406967,"p95":4.399999976158142,"max":7}},{"demand":[0,0],"frames":32,"work":{"mean":1.71875,"p95":3.899999976158142,"max":4.200000047683716}},{"demand":[1,1],"frames":32,"work":{"mean":5.087500017136335,"p95":5.5,"max":94.30000007152557}},{"demand":[2,2],"frames":32,"work":{"mean":1.6312500163912773,"p95":3.8000000715255737,"max":4.299999952316284}},{"demand":[2,0],"frames":32,"work":{"mean":2.0437499918043613,"p95":3.6999999284744263,"max":4.799999952316284}},{"demand":[2,2],"frames":32,"work":{"mean":1.7124999798834324,"p95":3.100000023841858,"max":4.299999952316284}}]},{"round":1,"codec":"avc1.420028","policy":"envelope","role":"relay","encoded":[128,96,0],"decoded":[224,224],"ingressDecodes":128,"creates":5,"generatedBytes":[248222,509962,0],"selectedBytes":[3510782,2330306],"switches":[{"index":32,"child":1,"layer":0,"ms":7.5,"bytes":25704},{"index":64,"child":0,"layer":0,"ms":5,"bytes":17134},{"index":96,"child":0,"layer":1,"ms":97.69999992847443,"bytes":56516},{"index":96,"child":1,"layer":1,"ms":98.29999995231628,"bytes":56516},{"index":128,"child":0,"layer":2,"ms":3.0999999046325684,"bytes":146641},{"index":128,"child":1,"layer":2,"ms":4.699999928474426,"bytes":146641},{"index":160,"child":1,"layer":0,"ms":5.799999952316284,"bytes":25704},{"index":192,"child":1,"layer":2,"ms":4.399999976158142,"bytes":146641}],"elapsedMs":7475,"metrics":{"management":{"mean":0.012053572173629488,"p95":0.10000002384185791,"max":0.19999992847442627},"input":{"mean":0.5120535668517862,"p95":1.5,"max":4.5},"encodeReady":{"mean":1.933482140302658,"p95":2.200000047683716,"max":95},"allReceivers":{"mean":2.3754464206951007,"p95":4.100000023841858,"max":98.29999995231628},"processing":{"mean":3.2549107069415704,"p95":4.399999976158142,"max":98.29999995231628}},"overFrameBudget":3,"phases":[{"demand":[2,2],"frames":32,"work":{"mean":1.7062499932944775,"p95":3,"max":7.5}},{"demand":[2,0],"frames":32,"work":{"mean":5.3624999932944775,"p95":5.5,"max":95.79999995231628}},{"demand":[0,0],"frames":32,"work":{"mean":1.7374999970197678,"p95":3.5,"max":5}},{"demand":[1,1],"frames":32,"work":{"mean":5.365624982863665,"p95":6.5,"max":98.29999995231628}},{"demand":[2,2],"frames":32,"work":{"mean":1.6187500059604645,"p95":2.799999952316284,"max":4.699999928474426}},{"demand":[2,0],"frames":32,"work":{"mean":5.3749999813735485,"p95":5.199999928474426,"max":97.89999997615814}},{"demand":[2,2],"frames":32,"work":{"mean":1.6187499947845936,"p95":3.299999952316284,"max":4.399999976158142}}]},{"round":2,"codec":"avc1.420028","policy":"envelope","role":"host","encoded":[224,192,160],"decoded":[224,224],"ingressDecodes":0,"creates":5,"generatedBytes":[384469,1010846,3257425],"selectedBytes":[3477453,2280622],"switches":[{"index":32,"child":1,"layer":0,"ms":2.1999999284744263,"bytes":17323},{"index":64,"child":0,"layer":0,"ms":3.100000023841858,"bytes":17323},{"index":96,"child":0,"layer":1,"ms":95.89999997615814,"bytes":79574},{"index":96,"child":1,"layer":1,"ms":96.5,"bytes":79574},{"index":128,"child":0,"layer":2,"ms":36,"bytes":146641},{"index":128,"child":1,"layer":2,"ms":37.59999990463257,"bytes":146641},{"index":160,"child":1,"layer":0,"ms":2.100000023841858,"bytes":17324},{"index":192,"child":1,"layer":2,"ms":14.799999952316284,"bytes":99623}],"elapsedMs":7467,"metrics":{"management":{"mean":0.00982142984867096,"p95":0.10000002384185791,"max":0.20000004768371582},"input":{"mean":0.1455357096024922,"p95":0.20000004768371582,"max":0.30000007152557373},"encodeReady":{"mean":4.7116071464759965,"p95":9,"max":94.5},"allReceivers":{"mean":5.80491071034755,"p95":11.100000023841858,"max":96.5},"processing":{"mean":5.80803571002824,"p95":11.100000023841858,"max":96.5}},"overFrameBudget":3,"phases":[{"demand":[2,2],"frames":32,"work":{"mean":7.578125,"p95":14.5,"max":53.299999952316284}},{"demand":[2,0],"frames":32,"work":{"mean":6.346875000745058,"p95":12.199999928474426,"max":12.5}},{"demand":[0,0],"frames":32,"work":{"mean":1.0062499791383743,"p95":2.1999999284744263,"max":3.100000023841858}},{"demand":[1,1],"frames":32,"work":{"mean":4.487499985843897,"p95":4.199999928474426,"max":96.5}},{"demand":[2,2],"frames":32,"work":{"mean":7.387500010430813,"p95":15.200000047683716,"max":37.59999990463257}},{"demand":[2,0],"frames":32,"work":{"mean":6.487500008195639,"p95":10.800000071525574,"max":11.100000023841858}},{"demand":[2,2],"frames":32,"work":{"mean":7.362499985843897,"p95":13.899999976158142,"max":14.799999952316284}}]},{"round":2,"codec":"avc1.420028","policy":"exact","role":"host","encoded":[96,32,160],"decoded":[224,224],"ingressDecodes":0,"creates":5,"generatedBytes":[185868,168726,3257425],"selectedBytes":[3477561,2312367],"switches":[{"index":32,"child":1,"layer":0,"ms":3,"bytes":32563},{"index":64,"child":0,"layer":0,"ms":6.5,"bytes":17323},{"index":96,"child":0,"layer":1,"ms":94.39999997615814,"bytes":79574},{"index":96,"child":1,"layer":1,"ms":95,"bytes":79574},{"index":128,"child":0,"layer":2,"ms":192.30000007152557,"bytes":146641},{"index":128,"child":1,"layer":2,"ms":193.89999997615814,"bytes":146641},{"index":160,"child":1,"layer":0,"ms":2.899999976158142,"bytes":32563},{"index":192,"child":1,"layer":2,"ms":11.399999976158142,"bytes":99623}],"elapsedMs":7465.899999976158,"metrics":{"management":{"mean":0.0035714294229234967,"p95":0,"max":0.10000002384185791},"input":{"mean":0.15625001224023954,"p95":0.20000004768371582,"max":0.30000007152557373},"encodeReady":{"mean":4.396874999361379,"p95":5.5,"max":189.79999995231628},"allReceivers":{"mean":5.53839286416769,"p95":7.600000023841858,"max":193.89999997615814},"processing":{"mean":5.539732149669102,"p95":7.600000023841858,"max":193.89999997615814}},"overFrameBudget":3,"phases":[{"demand":[2,2],"frames":32,"work":{"mean":8.243750017136335,"p95":9.5,"max":118.80000007152557}},{"demand":[2,0],"frames":32,"work":{"mean":4.528125017881393,"p95":7.5,"max":7.600000023841858}},{"demand":[0,0],"frames":32,"work":{"mean":1.146874975413084,"p95":2.1999999284744263,"max":6.5}},{"demand":[1,1],"frames":32,"work":{"mean":4.350000016391277,"p95":4.299999952316284,"max":95}},{"demand":[2,2],"frames":32,"work":{"mean":10.503125011920929,"p95":10.300000071525574,"max":193.89999997615814}},{"demand":[2,0],"frames":32,"work":{"mean":4.734375007450581,"p95":8,"max":9.300000071525574}},{"demand":[2,2],"frames":32,"work":{"mean":5.271875001490116,"p95":10.5,"max":11.399999976158142}}]},{"round":2,"codec":"avc1.420028","policy":"envelope","role":"relay","encoded":[128,96,0],"decoded":[224,224],"ingressDecodes":128,"creates":5,"generatedBytes":[248222,510247,0],"selectedBytes":[3511127,2330651],"switches":[{"index":32,"child":1,"layer":0,"ms":7.700000047683716,"bytes":25704},{"index":64,"child":0,"layer":0,"ms":4.600000023841858,"bytes":17134},{"index":96,"child":0,"layer":1,"ms":94.69999992847443,"bytes":56516},{"index":96,"child":1,"layer":1,"ms":95.29999995231628,"bytes":56516},{"index":128,"child":0,"layer":2,"ms":3.899999976158142,"bytes":146641},{"index":128,"child":1,"layer":2,"ms":6.299999952316284,"bytes":146641},{"index":160,"child":1,"layer":0,"ms":4.700000047683716,"bytes":25704},{"index":192,"child":1,"layer":2,"ms":5.100000023841858,"bytes":146641}],"elapsedMs":7467.199999928474,"metrics":{"management":{"mean":0.01249999925494194,"p95":0.10000002384185791,"max":0.20000004768371582},"input":{"mean":0.5035714272941861,"p95":1.600000023841858,"max":4.799999952316284},"encodeReady":{"mean":1.9209821447730064,"p95":2.200000047683716,"max":93.89999997615814},"allReceivers":{"mean":2.404910711305482,"p95":4.199999928474426,"max":95.29999995231628},"processing":{"mean":3.279910711305482,"p95":4.400000095367432,"max":98.20000004768372}},"overFrameBudget":3,"phases":[{"demand":[2,2],"frames":32,"work":{"mean":1.8875000067055225,"p95":3.0999999046325684,"max":14.5}},{"demand":[2,0],"frames":32,"work":{"mean":5.321874994784594,"p95":4.5,"max":98.20000004768372}},{"demand":[0,0],"frames":32,"work":{"mean":1.7312499918043613,"p95":4.199999928474426,"max":4.600000023841858}},{"demand":[1,1],"frames":32,"work":{"mean":5.284374989569187,"p95":5.799999952316284,"max":95.29999995231628}},{"demand":[2,2],"frames":32,"work":{"mean":1.668749999254942,"p95":2.899999976158142,"max":6.299999952316284}},{"demand":[2,0],"frames":32,"work":{"mean":5.359375014901161,"p95":4.700000047683716,"max":96}},{"demand":[2,2],"frames":32,"work":{"mean":1.7062499821186066,"p95":3.5999999046325684,"max":5.100000023841858}}]},{"round":2,"codec":"avc1.420028","policy":"exact","role":"relay","encoded":[96,32,0],"decoded":[224,224],"ingressDecodes":128,"creates":3,"generatedBytes":[185530,169743,0],"selectedBytes":[3510727,2330251],"switches":[{"index":32,"child":1,"layer":0,"ms":7.299999952316284,"bytes":25704},{"index":64,"child":0,"layer":0,"ms":4.5,"bytes":17134},{"index":96,"child":0,"layer":1,"ms":87.39999997615814,"bytes":56516},{"index":96,"child":1,"layer":1,"ms":88,"bytes":56516},{"index":128,"child":0,"layer":2,"ms":2.6999999284744263,"bytes":146641},{"index":128,"child":1,"layer":2,"ms":4.299999952316284,"bytes":146641},{"index":160,"child":1,"layer":0,"ms":5.399999976158142,"bytes":25704},{"index":192,"child":1,"layer":2,"ms":4.399999976158142,"bytes":146641}],"elapsedMs":7474.200000047684,"metrics":{"management":{"mean":0.004464285182101386,"p95":0,"max":0.19999992847442627},"input":{"mean":0.49642856206212727,"p95":1.5,"max":4.399999976158142},"encodeReady":{"mean":0.9008928548012461,"p95":1.399999976158142,"max":83.69999992847443},"allReceivers":{"mean":2.2392857037484646,"p95":3.5999999046325684,"max":88},"processing":{"mean":2.242857133171388,"p95":3.5999999046325684,"max":88}},"overFrameBudget":1,"phases":[{"demand":[2,2],"frames":32,"work":{"mean":1.749999988824129,"p95":3,"max":7.299999952316284}},{"demand":[2,0],"frames":32,"work":{"mean":2.156249985098839,"p95":4.5,"max":7.299999952316284}},{"demand":[0,0],"frames":32,"work":{"mean":1.7281249798834324,"p95":3.5999999046325684,"max":4.5}},{"demand":[1,1],"frames":32,"work":{"mean":4.8249999806284904,"p95":5.299999952316284,"max":88}},{"demand":[2,2],"frames":32,"work":{"mean":1.562500011175871,"p95":3.299999952316284,"max":4.299999952316284}},{"demand":[2,0],"frames":32,"work":{"mean":2.043749988079071,"p95":3.399999976158142,"max":5.399999976158142}},{"demand":[2,2],"frames":32,"work":{"mean":1.6343749985098839,"p95":3,"max":4.5}}]}],"workerSha256":"13bb853f7fc8237b51ecf8aa27fd39922320871d726b3931cdd7931f7f2a9851"} diff --git a/docs/research/data/node-local-feedback.json b/docs/research/data/node-local-feedback.json new file mode 100644 index 0000000..36aa2b6 --- /dev/null +++ b/docs/research/data/node-local-feedback.json @@ -0,0 +1 @@ +{"date":"2026-09-07","scope":"In-memory Pion GCC/TWCC payload feedback; not Browser, network, codec or quality acceptance","source":"internal/client/mediaedge/testdata/gcc_feedback_probe.go","sourceSha256":"156183418f6b778cda45e72c9e96774b9b1e5971bec481b6da596f40dba7a3cc","pionInterceptor":"v0.1.47","initialBitrate":5000000,"lowOutputCap":1250000,"linkPropagationMs":20,"linkMaxDelayMs":120,"feedbackMs":100,"pacer":"NoOpPacer","runs":[{"sourceCadence":"30fps frame bursts","rawSha256":"c5e24ebc5e43a9d82f0613716cf6baece41a67881eae6bf28aa16c85d51728ba","observations":[{"second":1.0003041,"arm":"low-layer-capped","phase":"high","capacityBps":6000000,"targetBps":5704154,"sourceBps":5000000,"sentBps":4990482,"receivedBps":4731361,"droppedPackets":0,"maxQueueMs":50.4},{"second":1.0003041,"arm":"follows-budget","phase":"high","capacityBps":6000000,"targetBps":5704154,"sourceBps":5000000,"sentBps":4990482,"receivedBps":4731361,"droppedPackets":0,"maxQueueMs":50.4},{"second":2.0052051,"arm":"low-layer-capped","phase":"high","capacityBps":6000000,"targetBps":6170446,"sourceBps":5000000,"sentBps":4977206,"receivedBps":4996313,"droppedPackets":0,"maxQueueMs":50.4},{"second":2.0052051,"arm":"follows-budget","phase":"high","capacityBps":6000000,"targetBps":6170446,"sourceBps":5000000,"sentBps":4977206,"receivedBps":4996313,"droppedPackets":0,"maxQueueMs":50.4},{"second":3.0101007,"arm":"low-layer-capped","phase":"high","capacityBps":6000000,"targetBps":6674342,"sourceBps":5000000,"sentBps":4977233,"receivedBps":5005893,"droppedPackets":0,"maxQueueMs":50.4},{"second":3.0101007,"arm":"follows-budget","phase":"high","capacityBps":6000000,"targetBps":6674342,"sourceBps":5000000,"sentBps":4977233,"receivedBps":5005893,"droppedPackets":0,"maxQueueMs":50.4},{"second":4.0102228,"arm":"low-layer-capped","phase":"high","capacityBps":6000000,"targetBps":7219386,"sourceBps":5000000,"sentBps":5000989,"receivedBps":5000989,"droppedPackets":0,"maxQueueMs":50.4},{"second":4.0102228,"arm":"follows-budget","phase":"high","capacityBps":6000000,"targetBps":7219386,"sourceBps":5000000,"sentBps":5000989,"receivedBps":5000989,"droppedPackets":0,"maxQueueMs":50.4},{"second":5.0102378,"arm":"low-layer-capped","phase":"constrained","capacityBps":500000,"targetBps":7679038,"sourceBps":5000000,"sentBps":5001524,"receivedBps":5001524,"droppedPackets":11,"maxQueueMs":116},{"second":5.0102378,"arm":"follows-budget","phase":"constrained","capacityBps":500000,"targetBps":7679038,"sourceBps":5000000,"sentBps":5001524,"receivedBps":5001524,"droppedPackets":11,"maxQueueMs":116},{"second":6.0150873,"arm":"low-layer-capped","phase":"constrained","capacityBps":500000,"targetBps":282094,"sourceBps":282094,"sentBps":1901180,"receivedBps":496790,"droppedPackets":157,"maxQueueMs":119.346},{"second":6.0150873,"arm":"follows-budget","phase":"constrained","capacityBps":500000,"targetBps":299577,"sourceBps":299577,"sentBps":1767428,"receivedBps":506344,"droppedPackets":142,"maxQueueMs":119.346},{"second":7.0151373,"arm":"low-layer-capped","phase":"constrained","capacityBps":500000,"targetBps":326557,"sourceBps":326557,"sentBps":297585,"receivedBps":297585,"droppedPackets":0,"maxQueueMs":58.4},{"second":7.0151373,"arm":"follows-budget","phase":"constrained","capacityBps":500000,"targetBps":346796,"sourceBps":346796,"sentBps":316784,"receivedBps":316784,"droppedPackets":0,"maxQueueMs":58.4},{"second":8.015183,"arm":"low-layer-capped","phase":"constrained","capacityBps":500000,"targetBps":416776,"sourceBps":416776,"sentBps":374382,"receivedBps":364783,"droppedPackets":0,"maxQueueMs":58.4},{"second":8.015183,"arm":"follows-budget","phase":"constrained","capacityBps":500000,"targetBps":442606,"sourceBps":442606,"sentBps":393582,"receivedBps":383982,"droppedPackets":0,"maxQueueMs":58.4},{"second":9.0154976,"arm":"low-layer-capped","phase":"constrained","capacityBps":500000,"targetBps":531920,"sourceBps":531920,"sentBps":470252,"receivedBps":460655,"droppedPackets":0,"maxQueueMs":70.079},{"second":9.0154976,"arm":"follows-budget","phase":"constrained","capacityBps":500000,"targetBps":555023,"sourceBps":555023,"sentBps":499043,"receivedBps":479849,"droppedPackets":0,"maxQueueMs":99.949},{"second":10.0203965,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":467581,"sourceBps":467581,"sentBps":468106,"receivedBps":477659,"droppedPackets":0,"maxQueueMs":88.323},{"second":10.0203965,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":472622,"sourceBps":472622,"sentBps":458553,"receivedBps":468106,"droppedPackets":0,"maxQueueMs":96.082},{"second":11.02057,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":505604,"sourceBps":505604,"sentBps":489515,"receivedBps":489515,"droppedPackets":0,"maxQueueMs":23.2},{"second":11.02057,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":511097,"sourceBps":511097,"sentBps":489515,"receivedBps":499113,"droppedPackets":0,"maxQueueMs":23.2},{"second":12.025383,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":547268,"sourceBps":547268,"sentBps":515916,"receivedBps":535024,"droppedPackets":0,"maxQueueMs":23.2},{"second":12.025383,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":553214,"sourceBps":553214,"sentBps":525470,"receivedBps":544578,"droppedPackets":0,"maxQueueMs":23.2},{"second":13.0301917,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":582913,"sourceBps":582913,"sentBps":563689,"receivedBps":563689,"droppedPackets":0,"maxQueueMs":24.8},{"second":13.0301917,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":589248,"sourceBps":589248,"sentBps":563689,"receivedBps":563689,"droppedPackets":0,"maxQueueMs":24.8},{"second":14.0351791,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":630510,"sourceBps":630510,"sentBps":630455,"receivedBps":601798,"droppedPackets":0,"maxQueueMs":24.8},{"second":14.0351791,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":637364,"sourceBps":637364,"sentBps":640008,"receivedBps":611350,"droppedPackets":0,"maxQueueMs":24.8},{"second":15.04011,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":682204,"sourceBps":682204,"sentBps":649596,"receivedBps":659149,"droppedPackets":0,"maxQueueMs":24.8},{"second":15.04011,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":689620,"sourceBps":689620,"sentBps":659149,"receivedBps":668702,"droppedPackets":0,"maxQueueMs":24.8},{"second":16.0401576,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":738250,"sourceBps":738250,"sentBps":719965,"receivedBps":710366,"droppedPackets":0,"maxQueueMs":24.8},{"second":16.0401576,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":746277,"sourceBps":746277,"sentBps":719965,"receivedBps":710366,"droppedPackets":0,"maxQueueMs":24.8},{"second":17.0403094,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":798841,"sourceBps":798841,"sentBps":767883,"receivedBps":767883,"droppedPackets":0,"maxQueueMs":24.8},{"second":17.0403094,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":807527,"sourceBps":807527,"sentBps":777481,"receivedBps":777481,"droppedPackets":0,"maxQueueMs":24.8},{"second":18.040407,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":864073,"sourceBps":864073,"sentBps":825519,"receivedBps":825519,"droppedPackets":0,"maxQueueMs":26.4},{"second":18.040407,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":873467,"sourceBps":873467,"sentBps":844717,"receivedBps":844717,"droppedPackets":0,"maxQueueMs":26.4},{"second":19.0405085,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":934631,"sourceBps":934631,"sentBps":902308,"receivedBps":892709,"droppedPackets":0,"maxQueueMs":26.4},{"second":19.0405085,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":944794,"sourceBps":944794,"sentBps":902308,"receivedBps":902308,"droppedPackets":0,"maxQueueMs":26.4},{"second":20.0405246,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":1010639,"sourceBps":1010639,"sentBps":959984,"receivedBps":969584,"droppedPackets":0,"maxQueueMs":26.4},{"second":20.0405246,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1021631,"sourceBps":1021631,"sentBps":979184,"receivedBps":969584,"droppedPackets":0,"maxQueueMs":26.4},{"second":21.0451813,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":1076469,"sourceBps":1076469,"sentBps":1041549,"receivedBps":1031994,"droppedPackets":0,"maxQueueMs":26.4},{"second":21.0451813,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1088177,"sourceBps":1088177,"sentBps":1051105,"receivedBps":1051105,"droppedPackets":0,"maxQueueMs":26.4},{"second":22.0453068,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":1164372,"sourceBps":1164372,"sentBps":1132657,"receivedBps":1132657,"droppedPackets":0,"maxQueueMs":28},{"second":22.0453068,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1177035,"sourceBps":1177035,"sentBps":1142256,"receivedBps":1132657,"droppedPackets":0,"maxQueueMs":28},{"second":23.0453602,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":1259939,"sourceBps":1250000,"sentBps":1219134,"receivedBps":1209535,"droppedPackets":0,"maxQueueMs":28},{"second":23.0453602,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1273642,"sourceBps":1273642,"sentBps":1228734,"receivedBps":1228734,"droppedPackets":0,"maxQueueMs":28},{"second":24.0501933,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":1362406,"sourceBps":1250000,"sentBps":1241997,"receivedBps":1241997,"droppedPackets":0,"maxQueueMs":28},{"second":24.0501933,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1377223,"sourceBps":1377223,"sentBps":1318427,"receivedBps":1318427,"droppedPackets":0,"maxQueueMs":28},{"second":25.0504186,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":1473660,"sourceBps":1250000,"sentBps":1247718,"receivedBps":1247718,"droppedPackets":0,"maxQueueMs":28},{"second":25.0504186,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1489686,"sourceBps":1489686,"sentBps":1439675,"receivedBps":1439675,"droppedPackets":0,"maxQueueMs":29.6},{"second":26.0551774,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":1593508,"sourceBps":1250000,"sentBps":1242089,"receivedBps":1251643,"droppedPackets":0,"maxQueueMs":28},{"second":26.0551774,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1610839,"sourceBps":1610839,"sentBps":1538279,"receivedBps":1528725,"droppedPackets":0,"maxQueueMs":29.6},{"second":27.0552808,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":1724297,"sourceBps":1250000,"sentBps":1247870,"receivedBps":1247870,"droppedPackets":0,"maxQueueMs":28},{"second":27.0552808,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1743050,"sourceBps":1743050,"sentBps":1670227,"receivedBps":1670227,"droppedPackets":0,"maxQueueMs":31.2},{"second":28.0601127,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":1864531,"sourceBps":1250000,"sentBps":1251552,"receivedBps":1270660,"droppedPackets":0,"maxQueueMs":28},{"second":28.0601127,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1884806,"sourceBps":1884806,"sentBps":1796121,"receivedBps":1824782,"droppedPackets":0,"maxQueueMs":31.2},{"second":29.0601266,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":2009526,"sourceBps":1250000,"sentBps":1247982,"receivedBps":1247982,"droppedPackets":0,"maxQueueMs":28},{"second":29.0601266,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":2038720,"sourceBps":2038720,"sentBps":1958372,"receivedBps":1939173,"droppedPackets":0,"maxQueueMs":32.8},{"second":30.0602887,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":2172958,"sourceBps":1250000,"sentBps":1247797,"receivedBps":1247797,"droppedPackets":0,"maxQueueMs":28},{"second":30.0602887,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":2204525,"sourceBps":2204525,"sentBps":2102059,"receivedBps":2102059,"droppedPackets":0,"maxQueueMs":32.8},{"second":31.0652641,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":2313792,"sourceBps":1250000,"sentBps":1241821,"receivedBps":1260926,"droppedPackets":0,"maxQueueMs":28},{"second":31.0652641,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":2347406,"sourceBps":2347406,"sentBps":2263936,"receivedBps":2283040,"droppedPackets":0,"maxQueueMs":34.4},{"second":32.0702508,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":2501969,"sourceBps":1250000,"sentBps":1289569,"receivedBps":1241807,"droppedPackets":0,"maxQueueMs":28},{"second":32.0702508,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":2538318,"sourceBps":2538318,"sentBps":2521824,"receivedBps":2464510,"droppedPackets":0,"maxQueueMs":36},{"second":33.0704674,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":2705451,"sourceBps":1250000,"sentBps":1247729,"receivedBps":1257327,"droppedPackets":0,"maxQueueMs":28},{"second":33.0704674,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":2744757,"sourceBps":2744757,"sentBps":2658624,"receivedBps":2639428,"droppedPackets":0,"maxQueueMs":36},{"second":34.0753313,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":2926383,"sourceBps":1250000,"sentBps":1241959,"receivedBps":1241959,"droppedPackets":0,"maxQueueMs":28},{"second":34.0753313,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":2968899,"sourceBps":2968899,"sentBps":2856506,"receivedBps":2856506,"droppedPackets":0,"maxQueueMs":37.6},{"second":35.0803056,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":3165356,"sourceBps":1250000,"sentBps":1251375,"receivedBps":1241822,"droppedPackets":0,"maxQueueMs":28},{"second":35.0803056,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":3211345,"sourceBps":3211345,"sentBps":3075899,"receivedBps":3075899,"droppedPackets":0,"maxQueueMs":39.2},{"second":36.0803088,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":3424900,"sourceBps":1250000,"sentBps":1247996,"receivedBps":1247996,"droppedPackets":0,"maxQueueMs":28},{"second":36.0803088,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":3474659,"sourceBps":3474659,"sentBps":3340789,"receivedBps":3331189,"droppedPackets":0,"maxQueueMs":40.8},{"second":37.0805343,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":3703446,"sourceBps":1250000,"sentBps":1247718,"receivedBps":1247718,"droppedPackets":0,"maxQueueMs":28},{"second":37.0805343,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":3757251,"sourceBps":3757251,"sentBps":3608786,"receivedBps":3589590,"droppedPackets":0,"maxQueueMs":42.4},{"second":38.085133,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":4005878,"sourceBps":1250000,"sentBps":1242287,"receivedBps":1251843,"droppedPackets":0,"maxQueueMs":28},{"second":38.085133,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":4064078,"sourceBps":4064078,"sentBps":3879758,"receivedBps":3870202,"droppedPackets":0,"maxQueueMs":44},{"second":39.0852271,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":4334341,"sourceBps":1250000,"sentBps":1247882,"receivedBps":1247882,"droppedPackets":0,"maxQueueMs":28},{"second":39.0852271,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":4397312,"sourceBps":4397312,"sentBps":4204404,"receivedBps":4194805,"droppedPackets":0,"maxQueueMs":45.6}]},{"rawSha256":"45b88c158d187d4071feccab5aaa40365fa19a6c831358ae3d12de76c8ca6f34","observations":[{"second":1.000381,"arm":"low-layer-capped","phase":"high","capacityBps":6000000,"targetBps":5000000,"sourceBps":5000000,"sentBps":4999695,"receivedBps":4865346,"droppedPackets":0,"maxQueueMs":24.8},{"second":1.000381,"arm":"follows-budget","phase":"high","capacityBps":6000000,"targetBps":5000000,"sourceBps":5000000,"sentBps":4999695,"receivedBps":4865346,"droppedPackets":0,"maxQueueMs":24.8},{"second":2.0051472,"arm":"low-layer-capped","phase":"high","capacityBps":6000000,"targetBps":5000000,"sourceBps":5000000,"sentBps":4996983,"receivedBps":5006537,"droppedPackets":0,"maxQueueMs":25.1},{"second":2.0051472,"arm":"follows-budget","phase":"high","capacityBps":6000000,"targetBps":5000000,"sourceBps":5000000,"sentBps":4996983,"receivedBps":5006537,"droppedPackets":0,"maxQueueMs":25.1},{"second":3.0054866,"arm":"low-layer-capped","phase":"high","capacityBps":6000000,"targetBps":5000000,"sourceBps":5000000,"sentBps":4999903,"receivedBps":4999903,"droppedPackets":0,"maxQueueMs":24.925},{"second":3.0054866,"arm":"follows-budget","phase":"high","capacityBps":6000000,"targetBps":5000000,"sourceBps":5000000,"sentBps":4999903,"receivedBps":4999903,"droppedPackets":0,"maxQueueMs":24.925},{"second":4.0101712,"arm":"low-layer-capped","phase":"high","capacityBps":6000000,"targetBps":5000000,"sourceBps":5000000,"sentBps":4997389,"receivedBps":4997389,"droppedPackets":0,"maxQueueMs":24.8},{"second":4.0101712,"arm":"follows-budget","phase":"high","capacityBps":6000000,"targetBps":5000000,"sourceBps":5000000,"sentBps":4997389,"receivedBps":4997389,"droppedPackets":0,"maxQueueMs":24.8},{"second":5.0102774,"arm":"low-layer-capped","phase":"constrained","capacityBps":500000,"targetBps":5000000,"sourceBps":5000000,"sentBps":5001068,"receivedBps":5001068,"droppedPackets":3,"maxQueueMs":110.996},{"second":5.0102774,"arm":"follows-budget","phase":"constrained","capacityBps":500000,"targetBps":5000000,"sourceBps":5000000,"sentBps":5001068,"receivedBps":5001068,"droppedPackets":3,"maxQueueMs":110.996},{"second":6.0152459,"arm":"low-layer-capped","phase":"constrained","capacityBps":500000,"targetBps":5000000,"sourceBps":5000000,"sentBps":4995977,"receivedBps":534942,"droppedPackets":470,"maxQueueMs":119.908},{"second":6.0152459,"arm":"follows-budget","phase":"constrained","capacityBps":500000,"targetBps":5000000,"sourceBps":5000000,"sentBps":4995977,"receivedBps":534942,"droppedPackets":470,"maxQueueMs":119.908},{"second":7.0200568,"arm":"low-layer-capped","phase":"constrained","capacityBps":500000,"targetBps":5000000,"sourceBps":5000000,"sentBps":5006315,"receivedBps":506363,"droppedPackets":472,"maxQueueMs":119.846},{"second":7.0200568,"arm":"follows-budget","phase":"constrained","capacityBps":500000,"targetBps":5000000,"sourceBps":5000000,"sentBps":5006315,"receivedBps":506363,"droppedPackets":472,"maxQueueMs":119.846},{"second":8.0200972,"arm":"low-layer-capped","phase":"constrained","capacityBps":500000,"targetBps":5000000,"sourceBps":5000000,"sentBps":5001397,"receivedBps":499179,"droppedPackets":469,"maxQueueMs":119.999},{"second":8.0200972,"arm":"follows-budget","phase":"constrained","capacityBps":500000,"targetBps":5000000,"sourceBps":5000000,"sentBps":5001397,"receivedBps":499179,"droppedPackets":469,"maxQueueMs":119.999},{"second":9.020417,"arm":"low-layer-capped","phase":"constrained","capacityBps":500000,"targetBps":5000000,"sourceBps":5000000,"sentBps":5000000,"receivedBps":499040,"droppedPackets":469,"maxQueueMs":119.944},{"second":9.020417,"arm":"follows-budget","phase":"constrained","capacityBps":500000,"targetBps":5000000,"sourceBps":5000000,"sentBps":5000000,"receivedBps":499040,"droppedPackets":469,"maxQueueMs":119.944},{"second":10.0253105,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":100000,"sourceBps":100000,"sentBps":143298,"receivedBps":162405,"droppedPackets":4,"maxQueueMs":117.856},{"second":10.0253105,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":100000,"sourceBps":100000,"sentBps":143298,"receivedBps":162405,"droppedPackets":4,"maxQueueMs":117.856},{"second":11.0254691,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":100000,"sourceBps":100000,"sentBps":105583,"receivedBps":95984,"droppedPackets":0,"maxQueueMs":21.6},{"second":11.0254691,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":100000,"sourceBps":100000,"sentBps":105583,"receivedBps":95984,"droppedPackets":0,"maxQueueMs":21.6},{"second":12.0304757,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":100000,"sourceBps":100000,"sentBps":95521,"receivedBps":105073,"droppedPackets":0,"maxQueueMs":21.6},{"second":12.0304757,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":100000,"sourceBps":100000,"sentBps":95521,"receivedBps":105073,"droppedPackets":0,"maxQueueMs":21.6},{"second":13.0350789,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":100000,"sourceBps":100000,"sentBps":105116,"receivedBps":95560,"droppedPackets":0,"maxQueueMs":21.6},{"second":13.0350789,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":100000,"sourceBps":100000,"sentBps":105116,"receivedBps":95560,"droppedPackets":0,"maxQueueMs":21.6},{"second":14.0351457,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":100000,"sourceBps":100000,"sentBps":95993,"receivedBps":105592,"droppedPackets":0,"maxQueueMs":21.6},{"second":14.0351457,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":100000,"sourceBps":100000,"sentBps":95993,"receivedBps":105592,"droppedPackets":0,"maxQueueMs":21.6},{"second":15.0354138,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":338621,"sourceBps":338621,"sentBps":220740,"receivedBps":211143,"droppedPackets":0,"maxQueueMs":21.6},{"second":15.0354138,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":338621,"sourceBps":338621,"sentBps":220740,"receivedBps":211143,"droppedPackets":0,"maxQueueMs":21.6},{"second":16.0402946,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":432173,"sourceBps":432173,"sentBps":382134,"receivedBps":382134,"droppedPackets":0,"maxQueueMs":21.6},{"second":16.0402946,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":432173,"sourceBps":432173,"sentBps":382134,"receivedBps":382134,"droppedPackets":0,"maxQueueMs":21.6},{"second":17.0451728,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":551572,"sourceBps":551572,"sentBps":477669,"receivedBps":477669,"droppedPackets":0,"maxQueueMs":21.6},{"second":17.0451728,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":551572,"sourceBps":551572,"sentBps":477669,"receivedBps":477669,"droppedPackets":0,"maxQueueMs":21.6},{"second":18.0501441,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":670437,"sourceBps":670437,"sentBps":611360,"receivedBps":601808,"droppedPackets":0,"maxQueueMs":21.6},{"second":18.0501441,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":670437,"sourceBps":670437,"sentBps":611360,"receivedBps":601808,"droppedPackets":0,"maxQueueMs":21.6},{"second":19.0501697,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":855662,"sourceBps":855662,"sentBps":777580,"receivedBps":777580,"droppedPackets":0,"maxQueueMs":21.6},{"second":19.0501697,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":855662,"sourceBps":855662,"sentBps":777580,"receivedBps":777580,"droppedPackets":0,"maxQueueMs":21.6},{"second":20.0501805,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":1092064,"sourceBps":1092064,"sentBps":988789,"receivedBps":979189,"droppedPackets":0,"maxQueueMs":21.6},{"second":20.0501805,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1092064,"sourceBps":1092064,"sentBps":988789,"receivedBps":979189,"droppedPackets":0,"maxQueueMs":21.6},{"second":21.0503118,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":1393780,"sourceBps":1250000,"sentBps":1209441,"receivedBps":1199842,"droppedPackets":0,"maxQueueMs":21.6},{"second":21.0503118,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1393780,"sourceBps":1393780,"sentBps":1247836,"receivedBps":1238237,"droppedPackets":0,"maxQueueMs":21.6},{"second":22.0504355,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":1778854,"sourceBps":1250000,"sentBps":1247845,"receivedBps":1257444,"droppedPackets":0,"maxQueueMs":21.6},{"second":22.0504355,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1377136,"sourceBps":1377136,"sentBps":1507013,"receivedBps":1507013,"droppedPackets":0,"maxQueueMs":21.6},{"second":23.0552004,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":2270316,"sourceBps":1250000,"sentBps":1251636,"receivedBps":1251636,"droppedPackets":0,"maxQueueMs":21.6},{"second":23.0552004,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1492999,"sourceBps":1492999,"sentBps":1433171,"receivedBps":1433171,"droppedPackets":0,"maxQueueMs":21.6},{"second":24.0552956,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":2897560,"sourceBps":1250000,"sentBps":1247881,"receivedBps":1247881,"droppedPackets":0,"maxQueueMs":21.6},{"second":24.0552956,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1598404,"sourceBps":1598404,"sentBps":1555051,"receivedBps":1545452,"droppedPackets":0,"maxQueueMs":21.6},{"second":25.0601539,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":3698100,"sourceBps":1250000,"sentBps":1251519,"receivedBps":1251519,"droppedPackets":0,"maxQueueMs":21.6},{"second":25.0601539,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1731640,"sourceBps":1731640,"sentBps":1652770,"receivedBps":1662323,"droppedPackets":0,"maxQueueMs":21.6},{"second":26.0602658,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":4719815,"sourceBps":1250000,"sentBps":1247860,"receivedBps":1247860,"droppedPackets":0,"maxQueueMs":21.6},{"second":26.0602658,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1574074,"sourceBps":1574074,"sentBps":1526229,"receivedBps":1526229,"droppedPackets":0,"maxQueueMs":21.6},{"second":27.0651212,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":6023810,"sourceBps":1250000,"sentBps":1251523,"receivedBps":1251523,"droppedPackets":0,"maxQueueMs":21.6},{"second":27.0651212,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1663222,"sourceBps":1663222,"sentBps":1624114,"receivedBps":1614560,"droppedPackets":0,"maxQueueMs":21.6},{"second":28.0654888,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":7688075,"sourceBps":1250000,"sentBps":1247541,"receivedBps":1247541,"droppedPackets":0,"maxQueueMs":21.6},{"second":28.0654888,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1508570,"sourceBps":1508570,"sentBps":1593014,"receivedBps":1602610,"droppedPackets":0,"maxQueueMs":21.6},{"second":29.0655245,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":9344901,"sourceBps":1250000,"sentBps":1247955,"receivedBps":1247955,"droppedPackets":0,"maxQueueMs":21.6},{"second":29.0655245,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1606341,"sourceBps":1606341,"sentBps":1555144,"receivedBps":1545544,"droppedPackets":0,"maxQueueMs":21.6},{"second":30.0703893,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":11926723,"sourceBps":1250000,"sentBps":1251511,"receivedBps":1251511,"droppedPackets":0,"maxQueueMs":21.6},{"second":30.0703893,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1711375,"sourceBps":1711375,"sentBps":1671866,"receivedBps":1671866,"droppedPackets":0,"maxQueueMs":21.6},{"second":31.0753136,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":15221854,"sourceBps":1250000,"sentBps":1251437,"receivedBps":1251437,"droppedPackets":0,"maxQueueMs":21.6},{"second":31.0753136,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1589603,"sourceBps":1589603,"sentBps":1538026,"receivedBps":1547579,"droppedPackets":0,"maxQueueMs":21.6},{"second":32.0801219,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":19427368,"sourceBps":1250000,"sentBps":1251582,"receivedBps":1251582,"droppedPackets":0,"maxQueueMs":21.6},{"second":32.0801219,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1719409,"sourceBps":1719409,"sentBps":1662406,"receivedBps":1662406,"droppedPackets":0,"maxQueueMs":21.6},{"second":33.0803874,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":24494383,"sourceBps":1250000,"sentBps":1247668,"receivedBps":1247668,"droppedPackets":0,"maxQueueMs":21.6},{"second":33.0803874,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1575457,"sourceBps":1575457,"sentBps":1535592,"receivedBps":1535592,"droppedPackets":0,"maxQueueMs":21.6},{"second":34.0850945,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":26486522,"sourceBps":1250000,"sentBps":1251708,"receivedBps":1251708,"droppedPackets":0,"maxQueueMs":21.6},{"second":34.0850945,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1703584,"sourceBps":1703584,"sentBps":1633909,"receivedBps":1633909,"droppedPackets":0,"maxQueueMs":21.6},{"second":35.0854041,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":28649501,"sourceBps":1250000,"sentBps":1247613,"receivedBps":1247613,"droppedPackets":0,"maxQueueMs":21.6},{"second":35.0854041,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1833139,"sourceBps":1833139,"sentBps":1765853,"receivedBps":1756256,"droppedPackets":0,"maxQueueMs":21.6},{"second":36.0901273,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":30991503,"sourceBps":1250000,"sentBps":1251688,"receivedBps":1251688,"droppedPackets":0,"maxQueueMs":21.6},{"second":36.0901273,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1666102,"sourceBps":1666102,"sentBps":1824980,"receivedBps":1834535,"droppedPackets":0,"maxQueueMs":21.6},{"second":37.0950875,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":33543023,"sourceBps":1250000,"sentBps":1251392,"receivedBps":1251392,"droppedPackets":0,"maxQueueMs":21.6},{"second":37.0950875,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1797358,"sourceBps":1797358,"sentBps":1719471,"receivedBps":1709918,"droppedPackets":0,"maxQueueMs":21.6},{"second":38.0952286,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":35706086,"sourceBps":1250000,"sentBps":1247823,"receivedBps":1247823,"droppedPackets":0,"maxQueueMs":21.6},{"second":38.0952286,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1828571,"sourceBps":1828571,"sentBps":1814144,"receivedBps":1814144,"droppedPackets":0,"maxQueueMs":21.6},{"second":39.0952899,"arm":"low-layer-capped","phase":"released","capacityBps":6000000,"targetBps":38621970,"sourceBps":1250000,"sentBps":1247923,"receivedBps":1247923,"droppedPackets":0,"maxQueueMs":21.6},{"second":39.0952899,"arm":"follows-budget","phase":"released","capacityBps":6000000,"targetBps":1868607,"sourceBps":1868607,"sentBps":1852686,"receivedBps":1852686,"droppedPackets":0,"maxQueueMs":21.6}]}]} diff --git a/docs/research/data/node-local-group.json b/docs/research/data/node-local-group.json new file mode 100644 index 0000000..28461a5 --- /dev/null +++ b/docs/research/data/node-local-group.json @@ -0,0 +1,506 @@ +{ + "passed": true, + "userAgent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) HeadlessChrome/152.0.0.0 Safari/537.36", + "results": [ + [ + [ + 0, + 640, + 360, + 33333 + ], + [ + 1, + 640, + 360, + 66667 + ], + [ + 2, + 640, + 360, + 100000 + ], + [ + 3, + 640, + 360, + 133333 + ], + [ + 4, + 640, + 360, + 166667 + ], + [ + 5, + 640, + 360, + 200000 + ], + [ + 6, + 640, + 360, + 233333 + ], + [ + 7, + 640, + 360, + 266667 + ], + [ + 8, + 640, + 360, + 300000 + ], + [ + 9, + 640, + 360, + 333333 + ], + [ + 10, + 640, + 360, + 366667 + ], + [ + 11, + 640, + 360, + 400000 + ], + [ + 12, + 640, + 360, + 433333 + ], + [ + 13, + 640, + 360, + 466667 + ], + [ + 14, + 640, + 360, + 500000 + ], + [ + 15, + 640, + 360, + 533333 + ], + [ + 16, + 160, + 90, + 566667 + ], + [ + 17, + 160, + 90, + 600000 + ], + [ + 18, + 160, + 90, + 633333 + ], + [ + 19, + 160, + 90, + 666667 + ], + [ + 20, + 160, + 90, + 700000 + ], + [ + 21, + 160, + 90, + 733333 + ], + [ + 22, + 160, + 90, + 766667 + ], + [ + 23, + 160, + 90, + 800000 + ], + [ + 24, + 320, + 180, + 833333 + ], + [ + 25, + 320, + 180, + 866667 + ], + [ + 26, + 320, + 180, + 900000 + ], + [ + 27, + 320, + 180, + 933333 + ], + [ + 28, + 320, + 180, + 966667 + ], + [ + 29, + 320, + 180, + 1000000 + ], + [ + 30, + 320, + 180, + 1033333 + ], + [ + 31, + 320, + 180, + 1066667 + ], + [ + 32, + 640, + 360, + 1100000 + ], + [ + 33, + 640, + 360, + 1133333 + ], + [ + 34, + 640, + 360, + 1166667 + ], + [ + 35, + 640, + 360, + 1200000 + ], + [ + 36, + 640, + 360, + 1233333 + ], + [ + 37, + 640, + 360, + 1266667 + ], + [ + 38, + 640, + 360, + 1300000 + ], + [ + 39, + 640, + 360, + 1333333 + ] + ], + [ + [ + 0, + 640, + 360, + 33333 + ], + [ + 1, + 640, + 360, + 66667 + ], + [ + 2, + 640, + 360, + 100000 + ], + [ + 3, + 640, + 360, + 133333 + ], + [ + 4, + 640, + 360, + 166667 + ], + [ + 5, + 640, + 360, + 200000 + ], + [ + 6, + 640, + 360, + 233333 + ], + [ + 7, + 640, + 360, + 266667 + ], + [ + 8, + 160, + 90, + 300000 + ], + [ + 9, + 160, + 90, + 333333 + ], + [ + 10, + 160, + 90, + 366667 + ], + [ + 11, + 160, + 90, + 400000 + ], + [ + 12, + 160, + 90, + 433333 + ], + [ + 13, + 160, + 90, + 466667 + ], + [ + 14, + 160, + 90, + 500000 + ], + [ + 15, + 160, + 90, + 533333 + ], + [ + 16, + 160, + 90, + 566667 + ], + [ + 17, + 160, + 90, + 600000 + ], + [ + 18, + 160, + 90, + 633333 + ], + [ + 19, + 160, + 90, + 666667 + ], + [ + 20, + 160, + 90, + 700000 + ], + [ + 21, + 160, + 90, + 733333 + ], + [ + 22, + 160, + 90, + 766667 + ], + [ + 23, + 160, + 90, + 800000 + ], + [ + 24, + 320, + 180, + 833333 + ], + [ + 25, + 320, + 180, + 866667 + ], + [ + 26, + 320, + 180, + 900000 + ], + [ + 27, + 320, + 180, + 933333 + ], + [ + 28, + 320, + 180, + 966667 + ], + [ + 29, + 320, + 180, + 1000000 + ], + [ + 30, + 320, + 180, + 1033333 + ], + [ + 31, + 320, + 180, + 1066667 + ], + [ + 32, + 640, + 360, + 1100000 + ], + [ + 33, + 640, + 360, + 1133333 + ], + [ + 34, + 640, + 360, + 1166667 + ], + [ + 35, + 640, + 360, + 1200000 + ], + [ + 36, + 640, + 360, + 1233333 + ], + [ + 37, + 640, + 360, + 1266667 + ], + [ + 38, + 640, + 360, + 1300000 + ], + [ + 39, + 640, + 360, + 1333333 + ] + ] + ], + "receivedSha256": "9c1bf7bddf4fe6d6a775931fa4d40d4019a19d736bf671f740ec440bf43b1765", + "scope": "Real Native VP8 -> shared forwarding -> two Pion PCs -> depacketized Chrome WebCodecs decode; not Browser WebRTC jitter-buffer or hardware acceptance", + "sources": { + "go.mod": "4f44d52dadc21573c9190f537082f762cb57e07e18db21312b144016a5d095ed", + "go.sum": "b601505515c6a66e97d8aa785f43269fe44dc30579c6dcf5e5a9e9a560941758", + "internal/media/encoded/packetizer.go": "0bf3fa342b306b4481c14265b3c340ce900d637baf9e9788ba54b28ca5195791", + "internal/media/forwarding/source.go": "ec70f86b375ca3185088dbf66e497e28070babf3ca4b23f9b81bcd3798d4735b", + "internal/media/forwarding/encoded_source.go": "42a17f730fd912738539e610ce66d1fc84a7e9851b4d3e238a246704b028eb94", + "internal/media/forwarding/output.go": "c0d5bedd7d72d6d559550a3727be6884d2aae69178035a87fcd42ba93c554ee3", + "internal/media/forwarding/transport.go": "aaed82c0d321c49c9abb7060c5567cb2b22ecbf415570b8a4fcc25908a55838e", + "internal/client/mediaedge/engine.go": "1bb2ea92753c60ca4eea0768e133ff9f91842f09677c57160b1c9855c42b5ff0", + "internal/client/mediaedge/source.go": "194b132f36bca3364b7dfdc8e2b8c3e0e9d2bed546ee00c135751f92dc1705de", + "internal/client/mediaedge/edge.go": "1b101da44d41457f5274371dcce96496933ddee47a4bd6fd2ecf3cf0499471a6", + "internal/client/mediaedge/group_fixture_test.go": "1dc12eb2521fc805ed86bf33ecef6bd49f850a9209d9cd153440c0dc26d766a1", + "native/capture/windows/encoded_group.fixture.cpp": "b2279e1ff8ef61d912dae0bee156d83de17034b48fb4002c8470d55e9a6d79d7" + } +} diff --git a/docs/research/data/node-local-vp8.json b/docs/research/data/node-local-vp8.json new file mode 100644 index 0000000..d1290e1 --- /dev/null +++ b/docs/research/data/node-local-vp8.json @@ -0,0 +1,3354 @@ +{ + "completed": true, + "scope": "CPU-only Native VP8 encode microbenchmark, not playback/network/H264 acceptance", + "input": "32 synthetic NV12 frames per size; generation and scaling excluded", + "profiles": [ + [ + 480, + 270, + 312500 + ], + [ + 960, + 540, + 1250000 + ], + [ + 1920, + 1080, + 5000000 + ] + ], + "fps": 30, + "threadsPerEncoder": 4, + "demandTrace": [ + "H/H", + "H/L", + "L/L", + "M/M", + "H/H", + "H/L", + "H/H" + ], + "secondsPerDemand": 2, + "envelope": "maximum-demand active set; no LiveKit demand detection/debounce emulation", + "warm": "retain at most three codec objects within the 14-second trace, stop inactive encoding", + "clockAblation": "change only encoder PTS to a continuous active-frame clock; not a production proposal", + "cpuMeasurement": "whole-window GetProcessTimes and QueryProcessCycleTime on Windows; cycles are not converted to milliseconds", + "sources": { + "native/capture/windows/encoded_variants.probe.cpp": "3eb7efff0fe3e9df174256d5da430905e76cc5748e671c8e25860e2207adab0a", + "native/capture/windows/vp8_encoder.cpp": "e8c840c935da9c81b6feaffcb8af7bd6726b63d3f6082733b4ead1a45c5d5e54" + }, + "host": { + "platform": "win32", + "release": "10.0.26200", + "cpu": "AMD Ryzen 7 9700X 8-Core Processor ", + "logicalCpus": 16 + }, + "results": [ + { + "round": 1, + "scenario": "high", + "scene": "still", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 4.338, + "workP95Ms": 5.425, + "workOverFrameBudget": 0, + "cpuMs": 703.125, + "cpuCyclesPerFrame": 40599505.111, + "cpuCoresAt30Fps": 0.234, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 90, + 0 + ], + "encodedBytes": [ + 0, + 0, + 1092435, + 0 + ] + }, + "inputResidentBytes": 134868992, + "activeResidentBytes": 211345408, + "stoppedResidentBytes": 136962048, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 4.338, + "workP95Ms": 5.425, + "workOverFrameBudget": 0, + "cpuMs": 703.125, + "cpuCyclesPerFrame": 40599505.111, + "cpuCoresAt30Fps": 0.234, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 90, + 0 + ], + "encodedBytes": [ + 0, + 0, + 1092435, + 0 + ] + } + ], + "transitions": [] + }, + { + "round": 1, + "scenario": "two-high", + "scene": "still", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 8.519, + "workP95Ms": 11.759, + "workOverFrameBudget": 0, + "cpuMs": 1218.75, + "cpuCyclesPerFrame": 78822674.633, + "cpuCoresAt30Fps": 0.406, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 90, + 90 + ], + "encodedBytes": [ + 0, + 0, + 1092435, + 1092435 + ] + }, + "inputResidentBytes": 134873088, + "activeResidentBytes": 286961664, + "stoppedResidentBytes": 135872512, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 8.519, + "workP95Ms": 11.759, + "workOverFrameBudget": 0, + "cpuMs": 1218.75, + "cpuCyclesPerFrame": 78822674.633, + "cpuCoresAt30Fps": 0.406, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 90, + 90 + ], + "encodedBytes": [ + 0, + 0, + 1092435, + 1092435 + ] + } + ], + "transitions": [] + }, + { + "round": 1, + "scenario": "high-low", + "scene": "still", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 4.79, + "workP95Ms": 6.514, + "workOverFrameBudget": 0, + "cpuMs": 937.5, + "cpuCyclesPerFrame": 44010347.422, + "cpuCoresAt30Fps": 0.312, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 0, + 90, + 0 + ], + "encodedBytes": [ + 72791, + 0, + 1092435, + 0 + ] + }, + "inputResidentBytes": 134868992, + "activeResidentBytes": 216936448, + "stoppedResidentBytes": 136728576, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 4.79, + "workP95Ms": 6.514, + "workOverFrameBudget": 0, + "cpuMs": 937.5, + "cpuCyclesPerFrame": 44010347.422, + "cpuCoresAt30Fps": 0.312, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 0, + 90, + 0 + ], + "encodedBytes": [ + 72791, + 0, + 1092435, + 0 + ] + } + ], + "transitions": [] + }, + { + "round": 1, + "scenario": "all", + "scene": "still", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 5.977, + "workP95Ms": 7.425, + "workOverFrameBudget": 0, + "cpuMs": 1109.375, + "cpuCyclesPerFrame": 54740335.456, + "cpuCoresAt30Fps": 0.37, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 90, + 90, + 0 + ], + "encodedBytes": [ + 72791, + 286127, + 1092435, + 0 + ] + }, + "inputResidentBytes": 134873088, + "activeResidentBytes": 236654592, + "stoppedResidentBytes": 135913472, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 5.977, + "workP95Ms": 7.425, + "workOverFrameBudget": 0, + "cpuMs": 1109.375, + "cpuCyclesPerFrame": 54740335.456, + "cpuCoresAt30Fps": 0.37, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 90, + 90, + 0 + ], + "encodedBytes": [ + 72791, + 286127, + 1092435, + 0 + ] + } + ], + "transitions": [] + }, + { + "round": 1, + "scenario": "mid-low", + "scene": "still", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 1.706, + "workP95Ms": 2.144, + "workOverFrameBudget": 0, + "cpuMs": 187.5, + "cpuCyclesPerFrame": 15686844.956, + "cpuCoresAt30Fps": 0.062, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 90, + 0, + 0 + ], + "encodedBytes": [ + 72791, + 286127, + 0, + 0 + ] + }, + "inputResidentBytes": 134868992, + "activeResidentBytes": 160935936, + "stoppedResidentBytes": 136105984, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 1.706, + "workP95Ms": 2.144, + "workOverFrameBudget": 0, + "cpuMs": 187.5, + "cpuCyclesPerFrame": 15686844.956, + "cpuCoresAt30Fps": 0.062, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 90, + 0, + 0 + ], + "encodedBytes": [ + 72791, + 286127, + 0, + 0 + ] + } + ], + "transitions": [] + }, + { + "round": 1, + "scenario": "low", + "scene": "still", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 0.458, + "workP95Ms": 0.611, + "workOverFrameBudget": 0, + "cpuMs": 109.375, + "cpuCyclesPerFrame": 3713051.878, + "cpuCoresAt30Fps": 0.036, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 0, + 0, + 0 + ], + "encodedBytes": [ + 72791, + 0, + 0, + 0 + ] + }, + "inputResidentBytes": 134873088, + "activeResidentBytes": 141131776, + "stoppedResidentBytes": 136880128, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 0.458, + "workP95Ms": 0.611, + "workOverFrameBudget": 0, + "cpuMs": 109.375, + "cpuCyclesPerFrame": 3713051.878, + "cpuCoresAt30Fps": 0.036, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 0, + 0, + 0 + ], + "encodedBytes": [ + 72791, + 0, + 0, + 0 + ] + } + ], + "transitions": [] + }, + { + "round": 1, + "scenario": "high", + "scene": "motion", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 6.497, + "workP95Ms": 9.354, + "workOverFrameBudget": 0, + "cpuMs": 1390.625, + "cpuCyclesPerFrame": 45229729.6, + "cpuCoresAt30Fps": 0.464, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 90, + 0 + ], + "encodedBytes": [ + 0, + 0, + 1528747, + 0 + ] + }, + "inputResidentBytes": 134873088, + "activeResidentBytes": 207904768, + "stoppedResidentBytes": 136970240, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 6.497, + "workP95Ms": 9.354, + "workOverFrameBudget": 0, + "cpuMs": 1390.625, + "cpuCyclesPerFrame": 45229729.6, + "cpuCoresAt30Fps": 0.464, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 90, + 0 + ], + "encodedBytes": [ + 0, + 0, + 1528747, + 0 + ] + } + ], + "transitions": [] + }, + { + "round": 1, + "scenario": "two-high", + "scene": "motion", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 12.741, + "workP95Ms": 18.444, + "workOverFrameBudget": 1, + "cpuMs": 1875, + "cpuCyclesPerFrame": 88019835.611, + "cpuCoresAt30Fps": 0.625, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 90, + 90 + ], + "encodedBytes": [ + 0, + 0, + 1528747, + 1528747 + ] + }, + "inputResidentBytes": 134873088, + "activeResidentBytes": 280084480, + "stoppedResidentBytes": 135872512, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 12.741, + "workP95Ms": 18.444, + "workOverFrameBudget": 1, + "cpuMs": 1875, + "cpuCyclesPerFrame": 88019835.611, + "cpuCoresAt30Fps": 0.625, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 90, + 90 + ], + "encodedBytes": [ + 0, + 0, + 1528747, + 1528747 + ] + } + ], + "transitions": [] + }, + { + "round": 1, + "scenario": "high-low", + "scene": "motion", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 7.088, + "workP95Ms": 9.902, + "workOverFrameBudget": 0, + "cpuMs": 1203.125, + "cpuCyclesPerFrame": 49396719.389, + "cpuCoresAt30Fps": 0.401, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 0, + 90, + 0 + ], + "encodedBytes": [ + 122802, + 0, + 1528747, + 0 + ] + }, + "inputResidentBytes": 134868992, + "activeResidentBytes": 213233664, + "stoppedResidentBytes": 136601600, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 7.088, + "workP95Ms": 9.902, + "workOverFrameBudget": 0, + "cpuMs": 1203.125, + "cpuCyclesPerFrame": 49396719.389, + "cpuCoresAt30Fps": 0.401, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 0, + 90, + 0 + ], + "encodedBytes": [ + 122802, + 0, + 1528747, + 0 + ] + } + ], + "transitions": [] + }, + { + "round": 1, + "scenario": "all", + "scene": "motion", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 8.681, + "workP95Ms": 11.961, + "workOverFrameBudget": 0, + "cpuMs": 1359.375, + "cpuCyclesPerFrame": 60308691.522, + "cpuCoresAt30Fps": 0.453, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 90, + 90, + 0 + ], + "encodedBytes": [ + 122802, + 506114, + 1528747, + 0 + ] + }, + "inputResidentBytes": 134873088, + "activeResidentBytes": 232017920, + "stoppedResidentBytes": 135905280, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 8.681, + "workP95Ms": 11.961, + "workOverFrameBudget": 0, + "cpuMs": 1359.375, + "cpuCyclesPerFrame": 60308691.522, + "cpuCoresAt30Fps": 0.453, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 90, + 90, + 0 + ], + "encodedBytes": [ + 122802, + 506114, + 1528747, + 0 + ] + } + ], + "transitions": [] + }, + { + "round": 1, + "scenario": "mid-low", + "scene": "motion", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 2.372, + "workP95Ms": 3.828, + "workOverFrameBudget": 0, + "cpuMs": 421.875, + "cpuCyclesPerFrame": 16465430.089, + "cpuCoresAt30Fps": 0.141, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 90, + 0, + 0 + ], + "encodedBytes": [ + 122802, + 506114, + 0, + 0 + ] + }, + "inputResidentBytes": 134873088, + "activeResidentBytes": 159875072, + "stoppedResidentBytes": 136130560, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 2.372, + "workP95Ms": 3.828, + "workOverFrameBudget": 0, + "cpuMs": 421.875, + "cpuCyclesPerFrame": 16465430.089, + "cpuCoresAt30Fps": 0.141, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 90, + 0, + 0 + ], + "encodedBytes": [ + 122802, + 506114, + 0, + 0 + ] + } + ], + "transitions": [] + }, + { + "round": 1, + "scenario": "low", + "scene": "motion", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 0.65, + "workP95Ms": 1.106, + "workOverFrameBudget": 0, + "cpuMs": 125, + "cpuCyclesPerFrame": 4339882.911, + "cpuCoresAt30Fps": 0.042, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 0, + 0, + 0 + ], + "encodedBytes": [ + 122802, + 0, + 0, + 0 + ] + }, + "inputResidentBytes": 134868992, + "activeResidentBytes": 140861440, + "stoppedResidentBytes": 136871936, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 0.65, + "workP95Ms": 1.106, + "workOverFrameBudget": 0, + "cpuMs": 125, + "cpuCyclesPerFrame": 4339882.911, + "cpuCoresAt30Fps": 0.042, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 0, + 0, + 0 + ], + "encodedBytes": [ + 122802, + 0, + 0, + 0 + ] + } + ], + "transitions": [] + }, + { + "round": 1, + "scenario": "envelope", + "scene": "motion", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 420, + "workMeanMs": 6.754, + "workP95Ms": 11.527, + "workOverFrameBudget": 0, + "cpuMs": 4796.875, + "cpuCyclesPerFrame": 47743354.238, + "cpuCoresAt30Fps": 0.343, + "createCount": 5, + "createMs": 15.062, + "encodedFrames": [ + 420, + 360, + 300, + 0 + ], + "encodedBytes": [ + 536510, + 1805256, + 4974370, + 0 + ] + }, + "inputResidentBytes": 134873088, + "activeResidentBytes": 232402944, + "stoppedResidentBytes": 135917568, + "phases": [ + { + "sourceFrames": 60, + "workMeanMs": 9.078, + "workP95Ms": 14.518, + "workOverFrameBudget": 0, + "cpuMs": 921.875, + "cpuCyclesPerFrame": 63839084.95, + "cpuCoresAt30Fps": 0.461, + "createCount": 3, + "createMs": 6.951, + "encodedFrames": [ + 60, + 60, + 60, + 0 + ], + "encodedBytes": [ + 76218, + 299124, + 1101559, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 8.588, + "workP95Ms": 9.615, + "workOverFrameBudget": 0, + "cpuMs": 890.625, + "cpuCyclesPerFrame": 60620952.533, + "cpuCoresAt30Fps": 0.445, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 60, + 60, + 60, + 0 + ], + "encodedBytes": [ + 78385, + 344891, + 906072, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 0.692, + "workP95Ms": 0.921, + "workOverFrameBudget": 0, + "cpuMs": 46.875, + "cpuCyclesPerFrame": 4506330.417, + "cpuCoresAt30Fps": 0.023, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 60, + 0, + 0, + 0 + ], + "encodedBytes": [ + 69518, + 0, + 0, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 2.433, + "workP95Ms": 3.754, + "workOverFrameBudget": 0, + "cpuMs": 234.375, + "cpuCyclesPerFrame": 17794100.967, + "cpuCoresAt30Fps": 0.117, + "createCount": 1, + "createMs": 2.396, + "encodedFrames": [ + 60, + 60, + 0, + 0 + ], + "encodedBytes": [ + 76630, + 294116, + 0, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 9.154, + "workP95Ms": 13.707, + "workOverFrameBudget": 0, + "cpuMs": 1031.25, + "cpuCyclesPerFrame": 65020105.567, + "cpuCoresAt30Fps": 0.516, + "createCount": 1, + "createMs": 5.714, + "encodedFrames": [ + 60, + 60, + 60, + 0 + ], + "encodedBytes": [ + 71017, + 300401, + 1117930, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 8.351, + "workP95Ms": 9.37, + "workOverFrameBudget": 0, + "cpuMs": 765.625, + "cpuCyclesPerFrame": 59696377.1, + "cpuCoresAt30Fps": 0.383, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 60, + 60, + 60, + 0 + ], + "encodedBytes": [ + 87244, + 283072, + 910931, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 8.979, + "workP95Ms": 12.151, + "workOverFrameBudget": 0, + "cpuMs": 906.25, + "cpuCyclesPerFrame": 62726528.133, + "cpuCoresAt30Fps": 0.453, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 60, + 60, + 60, + 0 + ], + "encodedBytes": [ + 77498, + 283652, + 937878, + 0 + ] + } + ], + "transitions": [ + { + "frame": 0, + "layer": 2, + "keyframeReadyMs": 20.636, + "keyframeBytes": 32308 + }, + { + "frame": 60, + "layer": 0, + "keyframeReadyMs": 20.003, + "keyframeBytes": 2812 + }, + { + "frame": 120, + "layer": 0, + "keyframeReadyMs": 5.596, + "keyframeBytes": 8361 + }, + { + "frame": 180, + "layer": 1, + "keyframeReadyMs": 7.176, + "keyframeBytes": 29822 + }, + { + "frame": 240, + "layer": 2, + "keyframeReadyMs": 22.111, + "keyframeBytes": 32268 + }, + { + "frame": 300, + "layer": 0, + "keyframeReadyMs": 19.459, + "keyframeBytes": 5957 + }, + { + "frame": 360, + "layer": 2, + "keyframeReadyMs": 18.184, + "keyframeBytes": 31246 + } + ] + }, + { + "round": 1, + "scenario": "exact-cold", + "scene": "motion", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 420, + "workMeanMs": 5.321, + "workP95Ms": 9.091, + "workOverFrameBudget": 0, + "cpuMs": 3640.625, + "cpuCyclesPerFrame": 37786592.14, + "cpuCoresAt30Fps": 0.26, + "createCount": 5, + "createMs": 12.874, + "encodedFrames": [ + 180, + 60, + 300, + 0 + ], + "encodedBytes": [ + 227182, + 294116, + 4976220, + 0 + ] + }, + "inputResidentBytes": 134868992, + "activeResidentBytes": 208228352, + "stoppedResidentBytes": 136589312, + "phases": [ + { + "sourceFrames": 60, + "workMeanMs": 6.862, + "workP95Ms": 12.544, + "workOverFrameBudget": 0, + "cpuMs": 421.875, + "cpuCyclesPerFrame": 48882755.5, + "cpuCoresAt30Fps": 0.211, + "createCount": 1, + "createMs": 4.352, + "encodedFrames": [ + 0, + 0, + 60, + 0 + ], + "encodedBytes": [ + 0, + 0, + 1101559, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 7.144, + "workP95Ms": 8.021, + "workOverFrameBudget": 0, + "cpuMs": 765.625, + "cpuCyclesPerFrame": 50520022.883, + "cpuCoresAt30Fps": 0.383, + "createCount": 1, + "createMs": 0.834, + "encodedFrames": [ + 60, + 0, + 60, + 0 + ], + "encodedBytes": [ + 72001, + 0, + 906072, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 0.694, + "workP95Ms": 0.863, + "workOverFrameBudget": 0, + "cpuMs": 46.875, + "cpuCyclesPerFrame": 5182470.8, + "cpuCoresAt30Fps": 0.023, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 60, + 0, + 0, + 0 + ], + "encodedBytes": [ + 83372, + 0, + 0, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 1.873, + "workP95Ms": 3.015, + "workOverFrameBudget": 0, + "cpuMs": 187.5, + "cpuCyclesPerFrame": 14114056.4, + "cpuCoresAt30Fps": 0.094, + "createCount": 1, + "createMs": 1.5, + "encodedFrames": [ + 0, + 60, + 0, + 0 + ], + "encodedBytes": [ + 0, + 294116, + 0, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 6.95, + "workP95Ms": 12.623, + "workOverFrameBudget": 0, + "cpuMs": 828.125, + "cpuCyclesPerFrame": 49941229.733, + "cpuCoresAt30Fps": 0.414, + "createCount": 1, + "createMs": 5.423, + "encodedFrames": [ + 0, + 0, + 60, + 0 + ], + "encodedBytes": [ + 0, + 0, + 1117930, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 7.051, + "workP95Ms": 8.164, + "workOverFrameBudget": 0, + "cpuMs": 765.625, + "cpuCyclesPerFrame": 49444201.517, + "cpuCoresAt30Fps": 0.383, + "createCount": 1, + "createMs": 0.765, + "encodedFrames": [ + 60, + 0, + 60, + 0 + ], + "encodedBytes": [ + 71809, + 0, + 910931, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 6.671, + "workP95Ms": 7.846, + "workOverFrameBudget": 0, + "cpuMs": 625, + "cpuCyclesPerFrame": 46421408.15, + "cpuCoresAt30Fps": 0.312, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 60, + 0 + ], + "encodedBytes": [ + 0, + 0, + 939728, + 0 + ] + } + ], + "transitions": [ + { + "frame": 0, + "layer": 2, + "keyframeReadyMs": 19.673, + "keyframeBytes": 32308 + }, + { + "frame": 60, + "layer": 0, + "keyframeReadyMs": 16.627, + "keyframeBytes": 8272 + }, + { + "frame": 120, + "layer": 0, + "keyframeReadyMs": 4.258, + "keyframeBytes": 4336 + }, + { + "frame": 180, + "layer": 1, + "keyframeReadyMs": 5.731, + "keyframeBytes": 29822 + }, + { + "frame": 240, + "layer": 2, + "keyframeReadyMs": 22.787, + "keyframeBytes": 32268 + }, + { + "frame": 300, + "layer": 0, + "keyframeReadyMs": 15.404, + "keyframeBytes": 8284 + }, + { + "frame": 360, + "layer": 2, + "keyframeReadyMs": 13.188, + "keyframeBytes": 31246 + } + ] + }, + { + "round": 1, + "scenario": "exact-warm", + "scene": "motion", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 420, + "workMeanMs": 4.989, + "workP95Ms": 7.914, + "workOverFrameBudget": 0, + "cpuMs": 3937.5, + "cpuCyclesPerFrame": 35658453.552, + "cpuCoresAt30Fps": 0.281, + "createCount": 3, + "createMs": 6.924, + "encodedFrames": [ + 180, + 60, + 300, + 0 + ], + "encodedBytes": [ + 207584, + 294116, + 4429337, + 0 + ] + }, + "inputResidentBytes": 134864896, + "activeResidentBytes": 231952384, + "stoppedResidentBytes": 135897088, + "phases": [ + { + "sourceFrames": 60, + "workMeanMs": 6.765, + "workP95Ms": 12.372, + "workOverFrameBudget": 0, + "cpuMs": 468.75, + "cpuCyclesPerFrame": 48188313.9, + "cpuCoresAt30Fps": 0.234, + "createCount": 1, + "createMs": 4.359, + "encodedFrames": [ + 0, + 0, + 60, + 0 + ], + "encodedBytes": [ + 0, + 0, + 1101559, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 7.15, + "workP95Ms": 8.607, + "workOverFrameBudget": 0, + "cpuMs": 593.75, + "cpuCyclesPerFrame": 50078138.633, + "cpuCoresAt30Fps": 0.297, + "createCount": 1, + "createMs": 0.814, + "encodedFrames": [ + 60, + 0, + 60, + 0 + ], + "encodedBytes": [ + 72001, + 0, + 906072, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 0.615, + "workP95Ms": 0.779, + "workOverFrameBudget": 0, + "cpuMs": 78.125, + "cpuCyclesPerFrame": 4742545.883, + "cpuCoresAt30Fps": 0.039, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 60, + 0, + 0, + 0 + ], + "encodedBytes": [ + 83372, + 0, + 0, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 1.816, + "workP95Ms": 3.055, + "workOverFrameBudget": 0, + "cpuMs": 281.25, + "cpuCyclesPerFrame": 13635436.517, + "cpuCoresAt30Fps": 0.141, + "createCount": 1, + "createMs": 1.752, + "encodedFrames": [ + 0, + 60, + 0, + 0 + ], + "encodedBytes": [ + 0, + 294116, + 0, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 5.291, + "workP95Ms": 6.083, + "workOverFrameBudget": 0, + "cpuMs": 671.875, + "cpuCyclesPerFrame": 39366895.9, + "cpuCoresAt30Fps": 0.336, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 60, + 0 + ], + "encodedBytes": [ + 0, + 0, + 606169, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 6.91, + "workP95Ms": 9.1, + "workOverFrameBudget": 0, + "cpuMs": 984.375, + "cpuCyclesPerFrame": 48596712.25, + "cpuCoresAt30Fps": 0.492, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 60, + 0, + 60, + 0 + ], + "encodedBytes": [ + 52211, + 0, + 935201, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 6.375, + "workP95Ms": 7.069, + "workOverFrameBudget": 0, + "cpuMs": 859.375, + "cpuCyclesPerFrame": 45001131.783, + "cpuCoresAt30Fps": 0.43, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 60, + 0 + ], + "encodedBytes": [ + 0, + 0, + 880336, + 0 + ] + } + ], + "transitions": [ + { + "frame": 0, + "layer": 2, + "keyframeReadyMs": 20.542, + "keyframeBytes": 32308 + }, + { + "frame": 60, + "layer": 0, + "keyframeReadyMs": 15.62, + "keyframeBytes": 8272 + }, + { + "frame": 120, + "layer": 0, + "keyframeReadyMs": 1.391, + "keyframeBytes": 4336 + }, + { + "frame": 180, + "layer": 1, + "keyframeReadyMs": 6.057, + "keyframeBytes": 29822 + }, + { + "frame": 240, + "layer": 2, + "keyframeReadyMs": 21.789, + "keyframeBytes": 228469 + }, + { + "frame": 300, + "layer": 0, + "keyframeReadyMs": 16.375, + "keyframeBytes": 22184 + }, + { + "frame": 360, + "layer": 2, + "keyframeReadyMs": 14.522, + "keyframeBytes": 43851 + } + ] + }, + { + "round": 1, + "scenario": "warm-clock-ablation", + "scene": "motion", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 420, + "workMeanMs": 5.166, + "workP95Ms": 8.458, + "workOverFrameBudget": 0, + "cpuMs": 3984.375, + "cpuCyclesPerFrame": 36539063.486, + "cpuCoresAt30Fps": 0.285, + "createCount": 3, + "createMs": 6.742, + "encodedFrames": [ + 180, + 60, + 300, + 0 + ], + "encodedBytes": [ + 223834, + 294116, + 4720828, + 0 + ] + }, + "inputResidentBytes": 134873088, + "activeResidentBytes": 232001536, + "stoppedResidentBytes": 135901184, + "phases": [ + { + "sourceFrames": 60, + "workMeanMs": 7.002, + "workP95Ms": 12.963, + "workOverFrameBudget": 0, + "cpuMs": 734.375, + "cpuCyclesPerFrame": 50002499.417, + "cpuCoresAt30Fps": 0.367, + "createCount": 1, + "createMs": 4.345, + "encodedFrames": [ + 0, + 0, + 60, + 0 + ], + "encodedBytes": [ + 0, + 0, + 1101559, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 6.952, + "workP95Ms": 8.135, + "workOverFrameBudget": 0, + "cpuMs": 531.25, + "cpuCyclesPerFrame": 48541372.583, + "cpuCoresAt30Fps": 0.266, + "createCount": 1, + "createMs": 0.784, + "encodedFrames": [ + 60, + 0, + 60, + 0 + ], + "encodedBytes": [ + 72001, + 0, + 906072, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 0.62, + "workP95Ms": 0.799, + "workOverFrameBudget": 0, + "cpuMs": 15.625, + "cpuCyclesPerFrame": 4771511.65, + "cpuCoresAt30Fps": 0.008, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 60, + 0, + 0, + 0 + ], + "encodedBytes": [ + 83372, + 0, + 0, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 1.822, + "workP95Ms": 3.111, + "workOverFrameBudget": 0, + "cpuMs": 218.75, + "cpuCyclesPerFrame": 13631252, + "cpuCoresAt30Fps": 0.109, + "createCount": 1, + "createMs": 1.613, + "encodedFrames": [ + 0, + 60, + 0, + 0 + ], + "encodedBytes": [ + 0, + 294116, + 0, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 6.618, + "workP95Ms": 8.134, + "workOverFrameBudget": 0, + "cpuMs": 796.875, + "cpuCyclesPerFrame": 45929727.467, + "cpuCoresAt30Fps": 0.398, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 60, + 0 + ], + "encodedBytes": [ + 0, + 0, + 946138, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 6.916, + "workP95Ms": 8.629, + "workOverFrameBudget": 0, + "cpuMs": 1000, + "cpuCyclesPerFrame": 48775678.983, + "cpuCoresAt30Fps": 0.5, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 60, + 0, + 60, + 0 + ], + "encodedBytes": [ + 68461, + 0, + 899110, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 6.235, + "workP95Ms": 7.212, + "workOverFrameBudget": 0, + "cpuMs": 687.5, + "cpuCyclesPerFrame": 44121402.3, + "cpuCoresAt30Fps": 0.344, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 60, + 0 + ], + "encodedBytes": [ + 0, + 0, + 867949, + 0 + ] + } + ], + "transitions": [ + { + "frame": 0, + "layer": 2, + "keyframeReadyMs": 20.744, + "keyframeBytes": 32308 + }, + { + "frame": 60, + "layer": 0, + "keyframeReadyMs": 16.727, + "keyframeBytes": 8272 + }, + { + "frame": 120, + "layer": 0, + "keyframeReadyMs": 1.366, + "keyframeBytes": 4336 + }, + { + "frame": 180, + "layer": 1, + "keyframeReadyMs": 6.018, + "keyframeBytes": 29822 + }, + { + "frame": 240, + "layer": 2, + "keyframeReadyMs": 13.235, + "keyframeBytes": 31079 + }, + { + "frame": 300, + "layer": 0, + "keyframeReadyMs": 15.961, + "keyframeBytes": 7185 + }, + { + "frame": 360, + "layer": 2, + "keyframeReadyMs": 14.751, + "keyframeBytes": 43851 + } + ] + }, + { + "round": 2, + "scenario": "low", + "scene": "still", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 0.446, + "workP95Ms": 0.63, + "workOverFrameBudget": 0, + "cpuMs": 140.625, + "cpuCyclesPerFrame": 3601520.767, + "cpuCoresAt30Fps": 0.047, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 0, + 0, + 0 + ], + "encodedBytes": [ + 72791, + 0, + 0, + 0 + ] + }, + "inputResidentBytes": 134873088, + "activeResidentBytes": 141135872, + "stoppedResidentBytes": 136880128, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 0.446, + "workP95Ms": 0.63, + "workOverFrameBudget": 0, + "cpuMs": 140.625, + "cpuCyclesPerFrame": 3601520.767, + "cpuCoresAt30Fps": 0.047, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 0, + 0, + 0 + ], + "encodedBytes": [ + 72791, + 0, + 0, + 0 + ] + } + ], + "transitions": [] + }, + { + "round": 2, + "scenario": "mid-low", + "scene": "still", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 1.668, + "workP95Ms": 2.118, + "workOverFrameBudget": 0, + "cpuMs": 328.125, + "cpuCyclesPerFrame": 15514971.444, + "cpuCoresAt30Fps": 0.109, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 90, + 0, + 0 + ], + "encodedBytes": [ + 72791, + 286127, + 0, + 0 + ] + }, + "inputResidentBytes": 134868992, + "activeResidentBytes": 160993280, + "stoppedResidentBytes": 136118272, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 1.668, + "workP95Ms": 2.118, + "workOverFrameBudget": 0, + "cpuMs": 328.125, + "cpuCyclesPerFrame": 15514971.444, + "cpuCoresAt30Fps": 0.109, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 90, + 0, + 0 + ], + "encodedBytes": [ + 72791, + 286127, + 0, + 0 + ] + } + ], + "transitions": [] + }, + { + "round": 2, + "scenario": "all", + "scene": "still", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 5.701, + "workP95Ms": 7.666, + "workOverFrameBudget": 0, + "cpuMs": 859.375, + "cpuCyclesPerFrame": 53234397.044, + "cpuCoresAt30Fps": 0.286, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 90, + 90, + 0 + ], + "encodedBytes": [ + 72791, + 286127, + 1092435, + 0 + ] + }, + "inputResidentBytes": 134868992, + "activeResidentBytes": 236564480, + "stoppedResidentBytes": 135909376, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 5.701, + "workP95Ms": 7.666, + "workOverFrameBudget": 0, + "cpuMs": 859.375, + "cpuCyclesPerFrame": 53234397.044, + "cpuCoresAt30Fps": 0.286, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 90, + 90, + 0 + ], + "encodedBytes": [ + 72791, + 286127, + 1092435, + 0 + ] + } + ], + "transitions": [] + }, + { + "round": 2, + "scenario": "high-low", + "scene": "still", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 4.564, + "workP95Ms": 5.585, + "workOverFrameBudget": 0, + "cpuMs": 1500, + "cpuCyclesPerFrame": 42623930.389, + "cpuCoresAt30Fps": 0.5, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 0, + 90, + 0 + ], + "encodedBytes": [ + 72791, + 0, + 1092435, + 0 + ] + }, + "inputResidentBytes": 134868992, + "activeResidentBytes": 216883200, + "stoppedResidentBytes": 136728576, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 4.564, + "workP95Ms": 5.585, + "workOverFrameBudget": 0, + "cpuMs": 1500, + "cpuCyclesPerFrame": 42623930.389, + "cpuCoresAt30Fps": 0.5, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 0, + 90, + 0 + ], + "encodedBytes": [ + 72791, + 0, + 1092435, + 0 + ] + } + ], + "transitions": [] + }, + { + "round": 2, + "scenario": "two-high", + "scene": "still", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 8.413, + "workP95Ms": 10.434, + "workOverFrameBudget": 0, + "cpuMs": 1906.25, + "cpuCyclesPerFrame": 76975458.089, + "cpuCoresAt30Fps": 0.635, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 90, + 90 + ], + "encodedBytes": [ + 0, + 0, + 1092435, + 1092435 + ] + }, + "inputResidentBytes": 134868992, + "activeResidentBytes": 286834688, + "stoppedResidentBytes": 135868416, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 8.413, + "workP95Ms": 10.434, + "workOverFrameBudget": 0, + "cpuMs": 1906.25, + "cpuCyclesPerFrame": 76975458.089, + "cpuCoresAt30Fps": 0.635, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 90, + 90 + ], + "encodedBytes": [ + 0, + 0, + 1092435, + 1092435 + ] + } + ], + "transitions": [] + }, + { + "round": 2, + "scenario": "high", + "scene": "still", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 4.172, + "workP95Ms": 4.936, + "workOverFrameBudget": 0, + "cpuMs": 750, + "cpuCyclesPerFrame": 39201885.767, + "cpuCoresAt30Fps": 0.25, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 90, + 0 + ], + "encodedBytes": [ + 0, + 0, + 1092435, + 0 + ] + }, + "inputResidentBytes": 134868992, + "activeResidentBytes": 211349504, + "stoppedResidentBytes": 136962048, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 4.172, + "workP95Ms": 4.936, + "workOverFrameBudget": 0, + "cpuMs": 750, + "cpuCyclesPerFrame": 39201885.767, + "cpuCoresAt30Fps": 0.25, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 90, + 0 + ], + "encodedBytes": [ + 0, + 0, + 1092435, + 0 + ] + } + ], + "transitions": [] + }, + { + "round": 2, + "scenario": "warm-clock-ablation", + "scene": "motion", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 420, + "workMeanMs": 5.124, + "workP95Ms": 8.36, + "workOverFrameBudget": 0, + "cpuMs": 3546.875, + "cpuCyclesPerFrame": 36452003.333, + "cpuCoresAt30Fps": 0.253, + "createCount": 3, + "createMs": 6.851, + "encodedFrames": [ + 180, + 60, + 300, + 0 + ], + "encodedBytes": [ + 223834, + 294116, + 4720828, + 0 + ] + }, + "inputResidentBytes": 134873088, + "activeResidentBytes": 231952384, + "stoppedResidentBytes": 136564736, + "phases": [ + { + "sourceFrames": 60, + "workMeanMs": 6.781, + "workP95Ms": 12.596, + "workOverFrameBudget": 0, + "cpuMs": 921.875, + "cpuCyclesPerFrame": 48471701.317, + "cpuCoresAt30Fps": 0.461, + "createCount": 1, + "createMs": 4.042, + "encodedFrames": [ + 0, + 0, + 60, + 0 + ], + "encodedBytes": [ + 0, + 0, + 1101559, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 6.943, + "workP95Ms": 8.254, + "workOverFrameBudget": 0, + "cpuMs": 734.375, + "cpuCyclesPerFrame": 48821966.233, + "cpuCoresAt30Fps": 0.367, + "createCount": 1, + "createMs": 0.816, + "encodedFrames": [ + 60, + 0, + 60, + 0 + ], + "encodedBytes": [ + 72001, + 0, + 906072, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 0.628, + "workP95Ms": 0.756, + "workOverFrameBudget": 0, + "cpuMs": 78.125, + "cpuCyclesPerFrame": 4837825.95, + "cpuCoresAt30Fps": 0.039, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 60, + 0, + 0, + 0 + ], + "encodedBytes": [ + 83372, + 0, + 0, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 1.847, + "workP95Ms": 3.114, + "workOverFrameBudget": 0, + "cpuMs": 203.125, + "cpuCyclesPerFrame": 13833621.117, + "cpuCoresAt30Fps": 0.102, + "createCount": 1, + "createMs": 1.994, + "encodedFrames": [ + 0, + 60, + 0, + 0 + ], + "encodedBytes": [ + 0, + 294116, + 0, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 6.472, + "workP95Ms": 7.365, + "workOverFrameBudget": 0, + "cpuMs": 562.5, + "cpuCyclesPerFrame": 45209399.133, + "cpuCoresAt30Fps": 0.281, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 60, + 0 + ], + "encodedBytes": [ + 0, + 0, + 946138, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 6.901, + "workP95Ms": 8.36, + "workOverFrameBudget": 0, + "cpuMs": 484.375, + "cpuCyclesPerFrame": 49000844.133, + "cpuCoresAt30Fps": 0.242, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 60, + 0, + 60, + 0 + ], + "encodedBytes": [ + 68461, + 0, + 899110, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 6.296, + "workP95Ms": 7.675, + "workOverFrameBudget": 0, + "cpuMs": 562.5, + "cpuCyclesPerFrame": 44988665.45, + "cpuCoresAt30Fps": 0.281, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 60, + 0 + ], + "encodedBytes": [ + 0, + 0, + 867949, + 0 + ] + } + ], + "transitions": [ + { + "frame": 0, + "layer": 2, + "keyframeReadyMs": 20.179, + "keyframeBytes": 32308 + }, + { + "frame": 60, + "layer": 0, + "keyframeReadyMs": 16.735, + "keyframeBytes": 8272 + }, + { + "frame": 120, + "layer": 0, + "keyframeReadyMs": 1.424, + "keyframeBytes": 4336 + }, + { + "frame": 180, + "layer": 1, + "keyframeReadyMs": 6.64, + "keyframeBytes": 29822 + }, + { + "frame": 240, + "layer": 2, + "keyframeReadyMs": 12.898, + "keyframeBytes": 31079 + }, + { + "frame": 300, + "layer": 0, + "keyframeReadyMs": 15.943, + "keyframeBytes": 7185 + }, + { + "frame": 360, + "layer": 2, + "keyframeReadyMs": 14.962, + "keyframeBytes": 43851 + } + ] + }, + { + "round": 2, + "scenario": "exact-warm", + "scene": "motion", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 420, + "workMeanMs": 4.97, + "workP95Ms": 8.023, + "workOverFrameBudget": 0, + "cpuMs": 4218.75, + "cpuCyclesPerFrame": 35567133.54, + "cpuCoresAt30Fps": 0.301, + "createCount": 3, + "createMs": 6.591, + "encodedFrames": [ + 180, + 60, + 300, + 0 + ], + "encodedBytes": [ + 207584, + 294116, + 4429337, + 0 + ] + }, + "inputResidentBytes": 134868992, + "activeResidentBytes": 231882752, + "stoppedResidentBytes": 136556544, + "phases": [ + { + "sourceFrames": 60, + "workMeanMs": 6.819, + "workP95Ms": 12.424, + "workOverFrameBudget": 0, + "cpuMs": 906.25, + "cpuCyclesPerFrame": 48590315.05, + "cpuCoresAt30Fps": 0.453, + "createCount": 1, + "createMs": 4.264, + "encodedFrames": [ + 0, + 0, + 60, + 0 + ], + "encodedBytes": [ + 0, + 0, + 1101559, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 7.033, + "workP95Ms": 8.023, + "workOverFrameBudget": 0, + "cpuMs": 765.625, + "cpuCyclesPerFrame": 49349932.833, + "cpuCoresAt30Fps": 0.383, + "createCount": 1, + "createMs": 0.769, + "encodedFrames": [ + 60, + 0, + 60, + 0 + ], + "encodedBytes": [ + 72001, + 0, + 906072, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 0.642, + "workP95Ms": 0.904, + "workOverFrameBudget": 0, + "cpuMs": 187.5, + "cpuCyclesPerFrame": 4973236.483, + "cpuCoresAt30Fps": 0.094, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 60, + 0, + 0, + 0 + ], + "encodedBytes": [ + 83372, + 0, + 0, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 1.81, + "workP95Ms": 3.052, + "workOverFrameBudget": 0, + "cpuMs": 187.5, + "cpuCyclesPerFrame": 13568736.9, + "cpuCoresAt30Fps": 0.094, + "createCount": 1, + "createMs": 1.558, + "encodedFrames": [ + 0, + 60, + 0, + 0 + ], + "encodedBytes": [ + 0, + 294116, + 0, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 5.13, + "workP95Ms": 5.738, + "workOverFrameBudget": 0, + "cpuMs": 281.25, + "cpuCyclesPerFrame": 38341723.533, + "cpuCoresAt30Fps": 0.141, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 60, + 0 + ], + "encodedBytes": [ + 0, + 0, + 606169, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 6.996, + "workP95Ms": 8.139, + "workOverFrameBudget": 0, + "cpuMs": 1156.25, + "cpuCyclesPerFrame": 49199003.283, + "cpuCoresAt30Fps": 0.578, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 60, + 0, + 60, + 0 + ], + "encodedBytes": [ + 52211, + 0, + 935201, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 6.356, + "workP95Ms": 7.421, + "workOverFrameBudget": 0, + "cpuMs": 734.375, + "cpuCyclesPerFrame": 44946986.7, + "cpuCoresAt30Fps": 0.367, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 60, + 0 + ], + "encodedBytes": [ + 0, + 0, + 880336, + 0 + ] + } + ], + "transitions": [ + { + "frame": 0, + "layer": 2, + "keyframeReadyMs": 20.71, + "keyframeBytes": 32308 + }, + { + "frame": 60, + "layer": 0, + "keyframeReadyMs": 16.579, + "keyframeBytes": 8272 + }, + { + "frame": 120, + "layer": 0, + "keyframeReadyMs": 1.421, + "keyframeBytes": 4336 + }, + { + "frame": 180, + "layer": 1, + "keyframeReadyMs": 5.77, + "keyframeBytes": 29822 + }, + { + "frame": 240, + "layer": 2, + "keyframeReadyMs": 21.558, + "keyframeBytes": 228469 + }, + { + "frame": 300, + "layer": 0, + "keyframeReadyMs": 16.927, + "keyframeBytes": 22184 + }, + { + "frame": 360, + "layer": 2, + "keyframeReadyMs": 14.679, + "keyframeBytes": 43851 + } + ] + }, + { + "round": 2, + "scenario": "exact-cold", + "scene": "motion", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 420, + "workMeanMs": 5.25, + "workP95Ms": 9.224, + "workOverFrameBudget": 0, + "cpuMs": 4687.5, + "cpuCyclesPerFrame": 37017168.495, + "cpuCoresAt30Fps": 0.335, + "createCount": 5, + "createMs": 11.468, + "encodedFrames": [ + 180, + 60, + 300, + 0 + ], + "encodedBytes": [ + 227182, + 294116, + 4976220, + 0 + ] + }, + "inputResidentBytes": 134873088, + "activeResidentBytes": 208179200, + "stoppedResidentBytes": 136597504, + "phases": [ + { + "sourceFrames": 60, + "workMeanMs": 6.798, + "workP95Ms": 11.715, + "workOverFrameBudget": 0, + "cpuMs": 1281.25, + "cpuCyclesPerFrame": 48112555.167, + "cpuCoresAt30Fps": 0.641, + "createCount": 1, + "createMs": 4.263, + "encodedFrames": [ + 0, + 0, + 60, + 0 + ], + "encodedBytes": [ + 0, + 0, + 1101559, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 7.123, + "workP95Ms": 7.631, + "workOverFrameBudget": 0, + "cpuMs": 828.125, + "cpuCyclesPerFrame": 50188931.65, + "cpuCoresAt30Fps": 0.414, + "createCount": 1, + "createMs": 0.838, + "encodedFrames": [ + 60, + 0, + 60, + 0 + ], + "encodedBytes": [ + 72001, + 0, + 906072, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 0.665, + "workP95Ms": 0.793, + "workOverFrameBudget": 0, + "cpuMs": 187.5, + "cpuCyclesPerFrame": 4975088.2, + "cpuCoresAt30Fps": 0.094, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 60, + 0, + 0, + 0 + ], + "encodedBytes": [ + 83372, + 0, + 0, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 1.822, + "workP95Ms": 3.16, + "workOverFrameBudget": 0, + "cpuMs": 250, + "cpuCyclesPerFrame": 13768425.733, + "cpuCoresAt30Fps": 0.125, + "createCount": 1, + "createMs": 1.561, + "encodedFrames": [ + 0, + 60, + 0, + 0 + ], + "encodedBytes": [ + 0, + 294116, + 0, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 6.832, + "workP95Ms": 11.655, + "workOverFrameBudget": 0, + "cpuMs": 546.875, + "cpuCyclesPerFrame": 48656173.883, + "cpuCoresAt30Fps": 0.273, + "createCount": 1, + "createMs": 4.051, + "encodedFrames": [ + 0, + 0, + 60, + 0 + ], + "encodedBytes": [ + 0, + 0, + 1117930, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 6.964, + "workP95Ms": 9.224, + "workOverFrameBudget": 0, + "cpuMs": 828.125, + "cpuCyclesPerFrame": 48336316.817, + "cpuCoresAt30Fps": 0.414, + "createCount": 1, + "createMs": 0.755, + "encodedFrames": [ + 60, + 0, + 60, + 0 + ], + "encodedBytes": [ + 71809, + 0, + 910931, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 6.549, + "workP95Ms": 7.418, + "workOverFrameBudget": 0, + "cpuMs": 765.625, + "cpuCyclesPerFrame": 45082688.017, + "cpuCoresAt30Fps": 0.383, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 60, + 0 + ], + "encodedBytes": [ + 0, + 0, + 939728, + 0 + ] + } + ], + "transitions": [ + { + "frame": 0, + "layer": 2, + "keyframeReadyMs": 20.893, + "keyframeBytes": 32308 + }, + { + "frame": 60, + "layer": 0, + "keyframeReadyMs": 15.545, + "keyframeBytes": 8272 + }, + { + "frame": 120, + "layer": 0, + "keyframeReadyMs": 3.742, + "keyframeBytes": 4336 + }, + { + "frame": 180, + "layer": 1, + "keyframeReadyMs": 6.294, + "keyframeBytes": 29822 + }, + { + "frame": 240, + "layer": 2, + "keyframeReadyMs": 19.761, + "keyframeBytes": 32268 + }, + { + "frame": 300, + "layer": 0, + "keyframeReadyMs": 16.61, + "keyframeBytes": 8284 + }, + { + "frame": 360, + "layer": 2, + "keyframeReadyMs": 14.382, + "keyframeBytes": 31246 + } + ] + }, + { + "round": 2, + "scenario": "envelope", + "scene": "motion", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 420, + "workMeanMs": 6.668, + "workP95Ms": 11.363, + "workOverFrameBudget": 1, + "cpuMs": 6109.375, + "cpuCyclesPerFrame": 46602515.583, + "cpuCoresAt30Fps": 0.436, + "createCount": 5, + "createMs": 12.898, + "encodedFrames": [ + 420, + 360, + 300, + 0 + ], + "encodedBytes": [ + 536510, + 1805256, + 4976220, + 0 + ] + }, + "inputResidentBytes": 134868992, + "activeResidentBytes": 232144896, + "stoppedResidentBytes": 136798208, + "phases": [ + { + "sourceFrames": 60, + "workMeanMs": 8.999, + "workP95Ms": 13.241, + "workOverFrameBudget": 0, + "cpuMs": 1140.625, + "cpuCyclesPerFrame": 63126475.317, + "cpuCoresAt30Fps": 0.57, + "createCount": 3, + "createMs": 6.636, + "encodedFrames": [ + 60, + 60, + 60, + 0 + ], + "encodedBytes": [ + 76218, + 299124, + 1101559, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 8.601, + "workP95Ms": 9.62, + "workOverFrameBudget": 1, + "cpuMs": 1000, + "cpuCyclesPerFrame": 60494139.317, + "cpuCoresAt30Fps": 0.5, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 60, + 60, + 60, + 0 + ], + "encodedBytes": [ + 78385, + 344891, + 906072, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 0.66, + "workP95Ms": 0.895, + "workOverFrameBudget": 0, + "cpuMs": 78.125, + "cpuCyclesPerFrame": 4365287.983, + "cpuCoresAt30Fps": 0.039, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 60, + 0, + 0, + 0 + ], + "encodedBytes": [ + 69518, + 0, + 0, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 2.407, + "workP95Ms": 3.725, + "workOverFrameBudget": 0, + "cpuMs": 328.125, + "cpuCyclesPerFrame": 17466828.283, + "cpuCoresAt30Fps": 0.164, + "createCount": 1, + "createMs": 1.56, + "encodedFrames": [ + 60, + 60, + 0, + 0 + ], + "encodedBytes": [ + 76630, + 294116, + 0, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 8.964, + "workP95Ms": 13.039, + "workOverFrameBudget": 0, + "cpuMs": 1125, + "cpuCyclesPerFrame": 62410755.083, + "cpuCoresAt30Fps": 0.562, + "createCount": 1, + "createMs": 4.702, + "encodedFrames": [ + 60, + 60, + 60, + 0 + ], + "encodedBytes": [ + 71017, + 300401, + 1117930, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 8.336, + "workP95Ms": 9.784, + "workOverFrameBudget": 0, + "cpuMs": 859.375, + "cpuCyclesPerFrame": 58408691.767, + "cpuCoresAt30Fps": 0.43, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 60, + 60, + 60, + 0 + ], + "encodedBytes": [ + 87244, + 283072, + 910931, + 0 + ] + }, + { + "sourceFrames": 60, + "workMeanMs": 8.71, + "workP95Ms": 9.7, + "workOverFrameBudget": 0, + "cpuMs": 1578.125, + "cpuCyclesPerFrame": 59945431.333, + "cpuCoresAt30Fps": 0.789, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 60, + 60, + 60, + 0 + ], + "encodedBytes": [ + 77498, + 283652, + 939728, + 0 + ] + } + ], + "transitions": [ + { + "frame": 0, + "layer": 2, + "keyframeReadyMs": 18.924, + "keyframeBytes": 32308 + }, + { + "frame": 60, + "layer": 0, + "keyframeReadyMs": 19.379, + "keyframeBytes": 2812 + }, + { + "frame": 120, + "layer": 0, + "keyframeReadyMs": 4.458, + "keyframeBytes": 8361 + }, + { + "frame": 180, + "layer": 1, + "keyframeReadyMs": 6.04, + "keyframeBytes": 29822 + }, + { + "frame": 240, + "layer": 2, + "keyframeReadyMs": 20.817, + "keyframeBytes": 32268 + }, + { + "frame": 300, + "layer": 0, + "keyframeReadyMs": 18.511, + "keyframeBytes": 5957 + }, + { + "frame": 360, + "layer": 2, + "keyframeReadyMs": 13.691, + "keyframeBytes": 31246 + } + ] + }, + { + "round": 2, + "scenario": "low", + "scene": "motion", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 0.652, + "workP95Ms": 0.978, + "workOverFrameBudget": 0, + "cpuMs": 93.75, + "cpuCyclesPerFrame": 4359299.067, + "cpuCoresAt30Fps": 0.031, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 0, + 0, + 0 + ], + "encodedBytes": [ + 122802, + 0, + 0, + 0 + ] + }, + "inputResidentBytes": 134873088, + "activeResidentBytes": 140865536, + "stoppedResidentBytes": 136876032, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 0.652, + "workP95Ms": 0.978, + "workOverFrameBudget": 0, + "cpuMs": 93.75, + "cpuCyclesPerFrame": 4359299.067, + "cpuCoresAt30Fps": 0.031, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 0, + 0, + 0 + ], + "encodedBytes": [ + 122802, + 0, + 0, + 0 + ] + } + ], + "transitions": [] + }, + { + "round": 2, + "scenario": "mid-low", + "scene": "motion", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 2.333, + "workP95Ms": 3.391, + "workOverFrameBudget": 0, + "cpuMs": 531.25, + "cpuCyclesPerFrame": 16267237.033, + "cpuCoresAt30Fps": 0.177, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 90, + 0, + 0 + ], + "encodedBytes": [ + 122802, + 506114, + 0, + 0 + ] + }, + "inputResidentBytes": 134868992, + "activeResidentBytes": 159858688, + "stoppedResidentBytes": 136118272, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 2.333, + "workP95Ms": 3.391, + "workOverFrameBudget": 0, + "cpuMs": 531.25, + "cpuCyclesPerFrame": 16267237.033, + "cpuCoresAt30Fps": 0.177, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 90, + 0, + 0 + ], + "encodedBytes": [ + 122802, + 506114, + 0, + 0 + ] + } + ], + "transitions": [] + }, + { + "round": 2, + "scenario": "all", + "scene": "motion", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 8.743, + "workP95Ms": 11.777, + "workOverFrameBudget": 0, + "cpuMs": 1890.625, + "cpuCyclesPerFrame": 60031697.044, + "cpuCoresAt30Fps": 0.63, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 90, + 90, + 0 + ], + "encodedBytes": [ + 122802, + 506114, + 1528747, + 0 + ] + }, + "inputResidentBytes": 134873088, + "activeResidentBytes": 232071168, + "stoppedResidentBytes": 135905280, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 8.743, + "workP95Ms": 11.777, + "workOverFrameBudget": 0, + "cpuMs": 1890.625, + "cpuCyclesPerFrame": 60031697.044, + "cpuCoresAt30Fps": 0.63, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 90, + 90, + 0 + ], + "encodedBytes": [ + 122802, + 506114, + 1528747, + 0 + ] + } + ], + "transitions": [] + }, + { + "round": 2, + "scenario": "high-low", + "scene": "motion", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 7.139, + "workP95Ms": 9.929, + "workOverFrameBudget": 0, + "cpuMs": 1265.625, + "cpuCyclesPerFrame": 49574775.556, + "cpuCoresAt30Fps": 0.422, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 0, + 90, + 0 + ], + "encodedBytes": [ + 122802, + 0, + 1528747, + 0 + ] + }, + "inputResidentBytes": 134873088, + "activeResidentBytes": 213245952, + "stoppedResidentBytes": 136605696, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 7.139, + "workP95Ms": 9.929, + "workOverFrameBudget": 0, + "cpuMs": 1265.625, + "cpuCyclesPerFrame": 49574775.556, + "cpuCoresAt30Fps": 0.422, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 90, + 0, + 90, + 0 + ], + "encodedBytes": [ + 122802, + 0, + 1528747, + 0 + ] + } + ], + "transitions": [] + }, + { + "round": 2, + "scenario": "two-high", + "scene": "motion", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 13.119, + "workP95Ms": 18.577, + "workOverFrameBudget": 1, + "cpuMs": 2234.375, + "cpuCyclesPerFrame": 89077219.1, + "cpuCoresAt30Fps": 0.745, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 90, + 90 + ], + "encodedBytes": [ + 0, + 0, + 1528747, + 1528747 + ] + }, + "inputResidentBytes": 134873088, + "activeResidentBytes": 279961600, + "stoppedResidentBytes": 135872512, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 13.119, + "workP95Ms": 18.577, + "workOverFrameBudget": 1, + "cpuMs": 2234.375, + "cpuCyclesPerFrame": 89077219.1, + "cpuCoresAt30Fps": 0.745, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 90, + 90 + ], + "encodedBytes": [ + 0, + 0, + 1528747, + 1528747 + ] + } + ], + "transitions": [] + }, + { + "round": 2, + "scenario": "high", + "scene": "motion", + "codec": "v1.17.0", + "summary": { + "sourceFrames": 90, + "workMeanMs": 6.533, + "workP95Ms": 9.993, + "workOverFrameBudget": 0, + "cpuMs": 718.75, + "cpuCyclesPerFrame": 45666274.333, + "cpuCoresAt30Fps": 0.24, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 90, + 0 + ], + "encodedBytes": [ + 0, + 0, + 1528747, + 0 + ] + }, + "inputResidentBytes": 134873088, + "activeResidentBytes": 207974400, + "stoppedResidentBytes": 136970240, + "phases": [ + { + "sourceFrames": 90, + "workMeanMs": 6.533, + "workP95Ms": 9.993, + "workOverFrameBudget": 0, + "cpuMs": 718.75, + "cpuCyclesPerFrame": 45666274.333, + "cpuCoresAt30Fps": 0.24, + "createCount": 0, + "createMs": 0, + "encodedFrames": [ + 0, + 0, + 90, + 0 + ], + "encodedBytes": [ + 0, + 0, + 1528747, + 0 + ] + } + ], + "transitions": [] + } + ] +} diff --git a/docs/research/data/webrtc-encoder-chain.json b/docs/research/data/webrtc-encoder-chain.json new file mode 100644 index 0000000..da9f844 --- /dev/null +++ b/docs/research/data/webrtc-encoder-chain.json @@ -0,0 +1,111 @@ +{ + "date": "2026-09-08", + "scope": "Local UDP Pion chain replays previously encoded synthetic VP8; separate forced-demand native derivation check; not integrated VSE-to-transport adaptation", + "webrtcCommit": "6f37672d358475cd17544121a12494da454d85fb", + "inputSha256": "8e0e9deb4c2a2d449200885cb433bc235e53a54b89758930bf32ec330b0a388a", + "sourceHealthy": { + "phase": "healthy", + "seconds": 6.00062, + "encodes": 187, + "hits": 173, + "a": { + "encoded": 180, + "decoded": 180, + "fps": 29.9969, + "kbps": 1776.14, + "errors": 0, + "keyframes": 2, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 8.0151, + "firstReducedMs": -1 + }, + "b": { + "encoded": 180, + "decoded": 180, + "fps": 29.9969, + "kbps": 1779.64, + "errors": 0, + "keyframes": 3, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 13.2214, + "firstReducedMs": -1 + }, + "aEncodeMs": 4, + "bEncodeMs": 2, + "aEncodeUsage": 36, + "bEncodeUsage": 34 + }, + "sourceIntegrity": { + "integrity": "pass", + "pooled": true, + "adjuster": false, + "groupAdjuster": true, + "unsafeSkip": false, + "encodes": 339, + "hits": 260, + "splits": 5, + "rateSplits": 1, + "dependencySplits": 2, + "created": 5, + "joins": 3, + "live": 0, + "maxLive": 2, + "callbacks": 599, + "samePayloadCompared": 260, + "decodedHashMismatches": 0, + "aWantsUpdates": 3, + "bWantsUpdates": 3, + "aMinPixels": 230400, + "bMinPixels": 230400 + }, + "chain": { + "inputFrames": 180, + "relayHops": 2, + "leaves": [ + { + "frames": 180, + "firstIndex": 0, + "lastIndex": 179 + }, + { + "frames": 167, + "firstIndex": 13, + "lastIndex": 179 + }, + { + "frames": 167, + "firstIndex": 13, + "lastIndex": 179 + } + ], + "derivationObserved": false, + "cleanup": true + }, + "localDerivation": { + "codec": "vp8", + "original": { + "width": 640, + "height": 360 + }, + "derived": { + "width": 320, + "height": 180 + }, + "compatibleChildren": 2, + "sameLiveProcess": true, + "firstConsumerRetirementPreservesOther": true, + "lastConsumerRetirementStopsProcess": true, + "automaticNetworkSelection": false + }, + "sourceSha256": { + "native/probes/encoder-pool/harness.cpp": "ea24fa618dbc2387969b85207fc9dd02b6927342f08d49cda649e25069e1f4c4", + "internal/client/mediaedge/relay_chain_fixture_test.go": "19e679e6e43dfd5f7bacab1e3d14b4269a894e07976c65fd7e33960cb8949683", + "internal/client/mediaedge/relay_fixture_test.go": "e8398294c80c2371cbf638e28a0a83f6f440eefc4fb1cf58f2e7179c45fcc17f" + } +} diff --git a/docs/research/data/webrtc-encoder-latency.json b/docs/research/data/webrtc-encoder-latency.json new file mode 100644 index 0000000..1febd49 --- /dev/null +++ b/docs/research/data/webrtc-encoder-latency.json @@ -0,0 +1,597 @@ +{ + "date": "2026-09-08", + "webrtcCommit": "6f37672d358475cd17544121a12494da454d85fb", + "scope": "Synthetic 45ms codec delay at640x360, proportional to pixels; not CPU/GPU stress or transport; registry mutex serializes physical groups; shared-pipeline B resource fields alias the one group", + "sourceSha256": { + "pool.h": "77d227db644184a99e5be7597f6f0f237b1de1665b49bb01a7e77235253d9f11", + "harness.cpp": "7bf17bc83f946c41038b702f7af734b3fd7ca0849f8e01fdc2b3c90069af2032" + }, + "runs": { + "realtime-healthy": [ + { + "phase": "healthy", + "seconds": 6.00026, + "encodes": 180, + "hits": 180, + "a": { + "encoded": 180, + "decoded": 180, + "fps": 29.9987, + "kbps": 1800.86, + "errors": 0, + "keyframes": 1, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 10.0527, + "firstReducedMs": -1 + }, + "b": { + "encoded": 180, + "decoded": 180, + "fps": 29.9987, + "kbps": 1800.86, + "errors": 0, + "keyframes": 1, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 10.2703, + "firstReducedMs": -1 + }, + "aEncodeMs": 3, + "bEncodeMs": 3, + "aEncodeUsage": 37, + "bEncodeUsage": 37, + "aCpuLimitedResolution": false, + "bCpuLimitedResolution": false, + "aCpuLimitedFramerate": false, + "bCpuLimitedFramerate": false + }, + { + "phase": "skipped-input", + "seconds": 3.00063, + "encodes": 92, + "hits": 87, + "a": { + "encoded": 90, + "decoded": 90, + "fps": 29.9937, + "kbps": 1866.85, + "errors": 0, + "keyframes": 1, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 4.7952, + "firstReducedMs": -1 + }, + "b": { + "encoded": 89, + "decoded": 89, + "fps": 29.6604, + "kbps": 1892.81, + "errors": 0, + "keyframes": 3, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 4.5921, + "firstReducedMs": -1 + }, + "aEncodeMs": 4, + "bEncodeMs": 4, + "aEncodeUsage": 28, + "bEncodeUsage": 28, + "aCpuLimitedResolution": false, + "bCpuLimitedResolution": false, + "aCpuLimitedFramerate": false, + "bCpuLimitedFramerate": false + }, + { + "phase": "retired-b", + "seconds": 2.00051, + "encodes": 60, + "hits": 0, + "a": { + "encoded": 60, + "decoded": 60, + "fps": 29.9923, + "kbps": 1925.24, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 3.6672, + "firstReducedMs": -1 + }, + "b": { + "encoded": 0, + "decoded": 0, + "fps": 0, + "kbps": 0, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": -1, + "firstReducedMs": -1 + }, + "aEncodeMs": 4, + "bEncodeMs": 4, + "aEncodeUsage": 23, + "bEncodeUsage": 28, + "aCpuLimitedResolution": false, + "bCpuLimitedResolution": false, + "aCpuLimitedFramerate": false, + "bCpuLimitedFramerate": false + }, + { + "integrity": "pass", + "pooled": true, + "adjuster": false, + "groupAdjuster": true, + "unsafeSkip": false, + "realtime": true, + "latencyProbe": false, + "encodes": 332, + "hits": 267, + "splits": 4, + "rateSplits": 1, + "dependencySplits": 1, + "created": 3, + "joins": 2, + "live": 0, + "maxLive": 2, + "callbacks": 599, + "samePayloadCompared": 267, + "decodedHashMismatches": 0, + "aWantsUpdates": 3, + "bWantsUpdates": 3, + "aMinPixels": 230400, + "bMinPixels": 230400 + } + ], + "latency-pooled": [ + { + "phase": "healthy", + "seconds": 6.00087, + "encodes": 180, + "hits": 180, + "a": { + "encoded": 180, + "decoded": 180, + "fps": 29.9956, + "kbps": 1801.99, + "errors": 0, + "keyframes": 1, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 10.3654, + "firstReducedMs": -1 + }, + "b": { + "encoded": 180, + "decoded": 180, + "fps": 29.9956, + "kbps": 1801.99, + "errors": 0, + "keyframes": 1, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 10.2066, + "firstReducedMs": -1 + }, + "aEncodeMs": 3, + "bEncodeMs": 4, + "aEncodeUsage": 36, + "bEncodeUsage": 36, + "aCpuLimitedResolution": false, + "bCpuLimitedResolution": false, + "aCpuLimitedFramerate": false, + "bCpuLimitedFramerate": false + }, + { + "phase": "encoder-delayed", + "seconds": 30.0637, + "encodes": 493, + "hits": 276, + "a": { + "encoded": 391, + "decoded": 391, + "fps": 13.0057, + "kbps": 939.521, + "errors": 0, + "keyframes": 146, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 49.4038, + "firstReducedMs": -1 + }, + "b": { + "encoded": 378, + "decoded": 378, + "fps": 12.5733, + "kbps": 942.246, + "errors": 0, + "keyframes": 149, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 49.5686, + "firstReducedMs": -1 + }, + "aEncodeMs": 101, + "bEncodeMs": 102, + "aEncodeUsage": 176, + "bEncodeUsage": 174, + "aCpuLimitedResolution": true, + "bCpuLimitedResolution": true, + "aCpuLimitedFramerate": false, + "bCpuLimitedFramerate": false + }, + { + "phase": "encoder-released", + "seconds": 30.0004, + "encodes": 900, + "hits": 900, + "a": { + "encoded": 900, + "decoded": 900, + "fps": 29.9996, + "kbps": 1890.12, + "errors": 0, + "keyframes": 4, + "width": 640, + "height": 360, + "minWidth": 480, + "minHeight": 270, + "firstFullMs": 19075.3, + "firstReducedMs": 6.3117 + }, + "b": { + "encoded": 900, + "decoded": 900, + "fps": 29.9996, + "kbps": 1890.12, + "errors": 0, + "keyframes": 4, + "width": 640, + "height": 360, + "minWidth": 480, + "minHeight": 270, + "firstFullMs": 19078, + "firstReducedMs": 8.0933 + }, + "aEncodeMs": 3, + "bEncodeMs": 3, + "aEncodeUsage": 23, + "bEncodeUsage": 23, + "aCpuLimitedResolution": false, + "bCpuLimitedResolution": false, + "aCpuLimitedFramerate": false, + "bCpuLimitedFramerate": false + }, + { + "phase": "skipped-input", + "seconds": 3.00077, + "encodes": 92, + "hits": 87, + "a": { + "encoded": 90, + "decoded": 90, + "fps": 29.9923, + "kbps": 1849.19, + "errors": 0, + "keyframes": 1, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 3.6455, + "firstReducedMs": -1 + }, + "b": { + "encoded": 89, + "decoded": 89, + "fps": 29.659, + "kbps": 1887.78, + "errors": 0, + "keyframes": 3, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 3.633, + "firstReducedMs": -1 + }, + "aEncodeMs": 3, + "bEncodeMs": 3, + "aEncodeUsage": 18, + "bEncodeUsage": 18, + "aCpuLimitedResolution": false, + "bCpuLimitedResolution": false, + "aCpuLimitedFramerate": false, + "bCpuLimitedFramerate": false + }, + { + "phase": "retired-b", + "seconds": 2.00037, + "encodes": 60, + "hits": 0, + "a": { + "encoded": 60, + "decoded": 60, + "fps": 29.9945, + "kbps": 1930.19, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 3.1658, + "firstReducedMs": -1 + }, + "b": { + "encoded": 0, + "decoded": 0, + "fps": 0, + "kbps": 0, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": -1, + "firstReducedMs": -1 + }, + "aEncodeMs": 3, + "bEncodeMs": 3, + "aEncodeUsage": 16, + "bEncodeUsage": 18, + "aCpuLimitedResolution": false, + "bCpuLimitedResolution": false, + "aCpuLimitedFramerate": false, + "bCpuLimitedFramerate": false + }, + { + "integrity": "pass", + "pooled": true, + "adjuster": false, + "groupAdjuster": true, + "unsafeSkip": false, + "realtime": true, + "latencyProbe": true, + "encodes": 1725, + "hits": 1443, + "splits": 218, + "rateSplits": 3, + "dependencySplits": 107, + "created": 115, + "joins": 110, + "live": 0, + "maxLive": 2, + "callbacks": 3168, + "samePayloadCompared": 1443, + "decodedHashMismatches": 0, + "aWantsUpdates": 7, + "bWantsUpdates": 7, + "aMinPixels": 138240, + "bMinPixels": 138240 + } + ], + "latency-shared-pipeline": [ + { + "phase": "healthy", + "seconds": 6.00081, + "encodes": 180, + "hits": 0, + "a": { + "encoded": 180, + "decoded": 180, + "fps": 29.996, + "kbps": 1799.7, + "errors": 0, + "keyframes": 1, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 7.8241, + "firstReducedMs": -1 + }, + "b": { + "encoded": 180, + "decoded": 180, + "fps": 29.996, + "kbps": 1799.7, + "errors": 0, + "keyframes": 1, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 10.9907, + "firstReducedMs": -1 + }, + "aEncodeMs": 5, + "bEncodeMs": 5, + "aEncodeUsage": 38, + "bEncodeUsage": 38, + "aCpuLimitedResolution": false, + "bCpuLimitedResolution": false, + "aCpuLimitedFramerate": false, + "bCpuLimitedFramerate": false + }, + { + "phase": "encoder-delayed", + "seconds": 30.0915, + "encodes": 484, + "hits": 0, + "a": { + "encoded": 484, + "decoded": 484, + "fps": 16.0843, + "kbps": 966.446, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 48.2413, + "firstReducedMs": -1 + }, + "b": { + "encoded": 484, + "decoded": 484, + "fps": 16.0843, + "kbps": 966.446, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 49.7266, + "firstReducedMs": -1 + }, + "aEncodeMs": 81, + "bEncodeMs": 81, + "aEncodeUsage": 165, + "bEncodeUsage": 165, + "aCpuLimitedResolution": false, + "bCpuLimitedResolution": false, + "aCpuLimitedFramerate": false, + "bCpuLimitedFramerate": false + }, + { + "phase": "encoder-released", + "seconds": 30.0012, + "encodes": 900, + "hits": 0, + "a": { + "encoded": 900, + "decoded": 900, + "fps": 29.9988, + "kbps": 1903.62, + "errors": 0, + "keyframes": 2, + "width": 640, + "height": 360, + "minWidth": 480, + "minHeight": 270, + "firstFullMs": 19042.4, + "firstReducedMs": 6.2564 + }, + "b": { + "encoded": 900, + "decoded": 900, + "fps": 29.9988, + "kbps": 1903.62, + "errors": 0, + "keyframes": 2, + "width": 640, + "height": 360, + "minWidth": 480, + "minHeight": 270, + "firstFullMs": 19045.5, + "firstReducedMs": 8.9076 + }, + "aEncodeMs": 5, + "bEncodeMs": 5, + "aEncodeUsage": 26, + "bEncodeUsage": 26, + "aCpuLimitedResolution": false, + "bCpuLimitedResolution": false, + "aCpuLimitedFramerate": false, + "bCpuLimitedFramerate": false + }, + { + "phase": "retired-b", + "seconds": 2.00086, + "encodes": 60, + "hits": 0, + "a": { + "encoded": 60, + "decoded": 60, + "fps": 29.9871, + "kbps": 1897.37, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 3.4665, + "firstReducedMs": -1 + }, + "b": { + "encoded": 0, + "decoded": 0, + "fps": 0, + "kbps": 0, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": -1, + "firstReducedMs": -1 + }, + "aEncodeMs": 3, + "bEncodeMs": 3, + "aEncodeUsage": 22, + "bEncodeUsage": 22, + "aCpuLimitedResolution": false, + "bCpuLimitedResolution": false, + "aCpuLimitedFramerate": false, + "bCpuLimitedFramerate": false + }, + { + "integrity": "pass", + "pooled": false, + "adjuster": true, + "groupAdjuster": false, + "unsafeSkip": false, + "realtime": true, + "latencyProbe": true, + "sharedPipeline": true, + "encodes": 1624, + "hits": 0, + "splits": 0, + "rateSplits": 0, + "dependencySplits": 0, + "created": 3, + "joins": 0, + "live": 0, + "maxLive": 1, + "callbacks": 1624, + "samePayloadCompared": 1564, + "decodedHashMismatches": 0, + "aWantsUpdates": 7, + "bWantsUpdates": 0, + "aMinPixels": 138240, + "bMinPixels": 230400 + } + ] + } +} diff --git a/docs/research/data/webrtc-encoder-pool.json b/docs/research/data/webrtc-encoder-pool.json new file mode 100644 index 0000000..1b6fcc6 --- /dev/null +++ b/docs/research/data/webrtc-encoder-pool.json @@ -0,0 +1,861 @@ +{ + "date": "2026-09-08", + "webrtcCommit": "6f37672d358475cd17544121a12494da454d85fb", + "sdk": "shiguredo m152.7977.0.2 windows x64", + "sdkSha256": "3250ee091eb745b9885482d92db352e6bcd5426940021ff90a095747d84d2137", + "scope": "Synthetic VP8 L1T1; two real VSEs; supplied encoder budgets, no transport or hardware; maintain-framerate; real WebRTC decode plus raw libvpx dependency hashes", + "sourceSha256": { + "harness.cpp": "acc6c02927da48412608074c7da0a44d33ae50fdb29455ca226df76a5cf94705", + "pool.h": "06ef768d8afc3e6f874d1b83cfd962604335dd53c8029b4478844b3f685d5a8a", + "CMakeLists.txt": "aaa438cc9e52ddeef0d15678a32a660b700202da82b29a1ab14dad018992681d" + }, + "runs": { + "final-stock": [ + { + "phase": "healthy", + "seconds": 6.00079, + "encodes": 360, + "hits": 0, + "a": { + "encoded": 180, + "decoded": 180, + "fps": 29.9961, + "kbps": 1791.92, + "errors": 0, + "keyframes": 1, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 8.2744, + "firstReducedMs": -1 + }, + "b": { + "encoded": 180, + "decoded": 180, + "fps": 29.9961, + "kbps": 1789.26, + "errors": 0, + "keyframes": 1, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 16.2003, + "firstReducedMs": -1 + }, + "aEncodeMs": 4, + "bEncodeMs": 3, + "aEncodeUsage": 37, + "bEncodeUsage": 36 + }, + { + "phase": "weak", + "seconds": 15.0013, + "encodes": 900, + "hits": 0, + "a": { + "encoded": 450, + "decoded": 450, + "fps": 29.9974, + "kbps": 1891.56, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 5.5833, + "firstReducedMs": -1 + }, + "b": { + "encoded": 154, + "decoded": 154, + "fps": 10.2658, + "kbps": 85.2843, + "errors": 0, + "keyframes": 2, + "width": 320, + "height": 180, + "minWidth": 320, + "minHeight": 180, + "firstFullMs": 10.0645, + "firstReducedMs": 4043.01 + }, + "aEncodeMs": 4, + "bEncodeMs": 1, + "aEncodeUsage": 13, + "bEncodeUsage": 64 + }, + { + "phase": "released", + "seconds": 25.0011, + "encodes": 1500, + "hits": 0, + "a": { + "encoded": 750, + "decoded": 750, + "fps": 29.9987, + "kbps": 1889.05, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 3.7557, + "firstReducedMs": -1 + }, + "b": { + "encoded": 744, + "decoded": 744, + "fps": 29.7587, + "kbps": 1839.76, + "errors": 0, + "keyframes": 2, + "width": 640, + "height": 360, + "minWidth": 320, + "minHeight": 180, + "firstFullMs": 24044.9, + "firstReducedMs": 7.0881 + }, + "aEncodeMs": 3, + "bEncodeMs": 3, + "aEncodeUsage": 11, + "bEncodeUsage": 64 + }, + { + "phase": "skipped-input", + "seconds": 3.00151, + "encodes": 179, + "hits": 0, + "a": { + "encoded": 90, + "decoded": 90, + "fps": 29.9849, + "kbps": 1894.58, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 4.7284, + "firstReducedMs": -1 + }, + "b": { + "encoded": 89, + "decoded": 89, + "fps": 29.6517, + "kbps": 1863.16, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 8.6938, + "firstReducedMs": -1 + }, + "aEncodeMs": 4, + "bEncodeMs": 3, + "aEncodeUsage": 11, + "bEncodeUsage": 64 + }, + { + "phase": "retired-b", + "seconds": 2.00084, + "encodes": 60, + "hits": 0, + "a": { + "encoded": 60, + "decoded": 60, + "fps": 29.9874, + "kbps": 1898.34, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 4.0861, + "firstReducedMs": -1 + }, + "b": { + "encoded": 0, + "decoded": 0, + "fps": 0, + "kbps": 0, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": -1, + "firstReducedMs": -1 + }, + "aEncodeMs": 4, + "bEncodeMs": 3, + "aEncodeUsage": 11, + "bEncodeUsage": 64 + }, + { + "integrity": "pass", + "pooled": false, + "adjuster": true, + "groupAdjuster": false, + "unsafeSkip": false, + "encodes": 2999, + "hits": 0, + "splits": 0, + "rateSplits": 0, + "dependencySplits": 0, + "created": 6, + "joins": 0, + "live": 0, + "maxLive": 2, + "callbacks": 2697, + "samePayloadCompared": 12, + "decodedHashMismatches": 0, + "aWantsUpdates": 3, + "bWantsUpdates": 11, + "aMinPixels": 230400, + "bMinPixels": 77760 + } + ], + "final-cache-only": [ + { + "phase": "healthy", + "seconds": 6.00043, + "encodes": 337, + "hits": 23, + "a": { + "encoded": 180, + "decoded": 180, + "fps": 29.9978, + "kbps": 1726.01, + "errors": 0, + "keyframes": 3, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 8.445, + "firstReducedMs": -1 + }, + "b": { + "encoded": 180, + "decoded": 180, + "fps": 29.9978, + "kbps": 1755.78, + "errors": 0, + "keyframes": 3, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 14.006, + "firstReducedMs": -1 + }, + "aEncodeMs": 3, + "bEncodeMs": 3, + "aEncodeUsage": 37, + "bEncodeUsage": 36 + }, + { + "phase": "skipped-input", + "seconds": 3.00103, + "encodes": 179, + "hits": 0, + "a": { + "encoded": 90, + "decoded": 90, + "fps": 29.9897, + "kbps": 1881.13, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 3.7504, + "firstReducedMs": -1 + }, + "b": { + "encoded": 89, + "decoded": 89, + "fps": 29.6565, + "kbps": 1889.11, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 7.9936, + "firstReducedMs": -1 + }, + "aEncodeMs": 4, + "bEncodeMs": 4, + "aEncodeUsage": 27, + "bEncodeUsage": 27 + }, + { + "phase": "retired-b", + "seconds": 2.0012, + "encodes": 60, + "hits": 0, + "a": { + "encoded": 60, + "decoded": 60, + "fps": 29.982, + "kbps": 1902.96, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 4.2367, + "firstReducedMs": -1 + }, + "b": { + "encoded": 0, + "decoded": 0, + "fps": 0, + "kbps": 0, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": -1, + "firstReducedMs": -1 + }, + "aEncodeMs": 4, + "bEncodeMs": 4, + "aEncodeUsage": 23, + "bEncodeUsage": 27 + }, + { + "integrity": "pass", + "pooled": true, + "adjuster": true, + "groupAdjuster": false, + "unsafeSkip": false, + "encodes": 576, + "hits": 23, + "splits": 4, + "rateSplits": 2, + "dependencySplits": 2, + "created": 6, + "joins": 2, + "live": 0, + "maxLive": 2, + "callbacks": 599, + "samePayloadCompared": 23, + "decodedHashMismatches": 0, + "aWantsUpdates": 3, + "bWantsUpdates": 3, + "aMinPixels": 230400, + "bMinPixels": 230400 + } + ], + "final-group": [ + { + "phase": "healthy", + "seconds": 6.00104, + "encodes": 187, + "hits": 173, + "a": { + "encoded": 180, + "decoded": 180, + "fps": 29.9948, + "kbps": 1778.06, + "errors": 0, + "keyframes": 2, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 7.561, + "firstReducedMs": -1 + }, + "b": { + "encoded": 180, + "decoded": 180, + "fps": 29.9948, + "kbps": 1781.56, + "errors": 0, + "keyframes": 3, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 12.779, + "firstReducedMs": -1 + }, + "aEncodeMs": 3, + "bEncodeMs": 2, + "aEncodeUsage": 36, + "bEncodeUsage": 34 + }, + { + "phase": "weak", + "seconds": 15.0012, + "encodes": 900, + "hits": 0, + "a": { + "encoded": 450, + "decoded": 450, + "fps": 29.9977, + "kbps": 1894.91, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 4.6906, + "firstReducedMs": -1 + }, + "b": { + "encoded": 142, + "decoded": 142, + "fps": 9.46593, + "kbps": 95.1078, + "errors": 0, + "keyframes": 3, + "width": 320, + "height": 180, + "minWidth": 320, + "minHeight": 180, + "firstFullMs": 13.8117, + "firstReducedMs": 4043.56 + }, + "aEncodeMs": 3, + "bEncodeMs": 1, + "aEncodeUsage": 13, + "bEncodeUsage": 64 + }, + { + "phase": "released", + "seconds": 25.0008, + "encodes": 1472, + "hits": 28, + "a": { + "encoded": 750, + "decoded": 750, + "fps": 29.999, + "kbps": 1890.79, + "errors": 0, + "keyframes": 1, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 3.6351, + "firstReducedMs": -1 + }, + "b": { + "encoded": 744, + "decoded": 744, + "fps": 29.759, + "kbps": 1861.66, + "errors": 0, + "keyframes": 3, + "width": 640, + "height": 360, + "minWidth": 320, + "minHeight": 180, + "firstFullMs": 24046.7, + "firstReducedMs": 6.7391 + }, + "aEncodeMs": 4, + "bEncodeMs": 3, + "aEncodeUsage": 10, + "bEncodeUsage": 64 + }, + { + "phase": "skipped-input", + "seconds": 3.00126, + "encodes": 92, + "hits": 87, + "a": { + "encoded": 90, + "decoded": 90, + "fps": 29.9874, + "kbps": 1872.53, + "errors": 0, + "keyframes": 1, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 3.759, + "firstReducedMs": -1 + }, + "b": { + "encoded": 89, + "decoded": 89, + "fps": 29.6543, + "kbps": 1916.06, + "errors": 0, + "keyframes": 3, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 5.3314, + "firstReducedMs": -1 + }, + "aEncodeMs": 3, + "bEncodeMs": 1, + "aEncodeUsage": 11, + "bEncodeUsage": 64 + }, + { + "phase": "retired-b", + "seconds": 2.0009, + "encodes": 60, + "hits": 0, + "a": { + "encoded": 60, + "decoded": 60, + "fps": 29.9865, + "kbps": 1891.39, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 3.7108, + "firstReducedMs": -1 + }, + "b": { + "encoded": 0, + "decoded": 0, + "fps": 0, + "kbps": 0, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": -1, + "firstReducedMs": -1 + }, + "aEncodeMs": 4, + "bEncodeMs": 1, + "aEncodeUsage": 11, + "bEncodeUsage": 64 + }, + { + "integrity": "pass", + "pooled": true, + "adjuster": false, + "groupAdjuster": true, + "unsafeSkip": false, + "encodes": 2711, + "hits": 288, + "splits": 7, + "rateSplits": 2, + "dependencySplits": 3, + "created": 11, + "joins": 4, + "live": 0, + "maxLive": 2, + "callbacks": 2685, + "samePayloadCompared": 288, + "decodedHashMismatches": 0, + "aWantsUpdates": 3, + "bWantsUpdates": 11, + "aMinPixels": 230400, + "bMinPixels": 77760 + } + ], + "raw-safe": [ + { + "phase": "healthy", + "seconds": 6.00048, + "encodes": 187, + "hits": 173, + "a": { + "encoded": 180, + "decoded": 180, + "fps": 29.9976, + "kbps": 1778.36, + "errors": 0, + "keyframes": 2, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 8.3785, + "firstReducedMs": -1 + }, + "b": { + "encoded": 180, + "decoded": 180, + "fps": 29.9976, + "kbps": 1781.86, + "errors": 0, + "keyframes": 3, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 13.8511, + "firstReducedMs": -1 + }, + "aEncodeMs": 3, + "bEncodeMs": 1, + "aEncodeUsage": 37, + "bEncodeUsage": 34 + }, + { + "phase": "skipped-input", + "seconds": 3.00072, + "encodes": 92, + "hits": 87, + "a": { + "encoded": 90, + "decoded": 90, + "fps": 29.9928, + "kbps": 1861.75, + "errors": 0, + "keyframes": 1, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 4.6064, + "firstReducedMs": -1 + }, + "b": { + "encoded": 89, + "decoded": 89, + "fps": 29.6596, + "kbps": 1887.75, + "errors": 0, + "keyframes": 3, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 6.7218, + "firstReducedMs": -1 + }, + "aEncodeMs": 3, + "bEncodeMs": 2, + "aEncodeUsage": 27, + "bEncodeUsage": 24 + }, + { + "phase": "retired-b", + "seconds": 2.001, + "encodes": 60, + "hits": 0, + "a": { + "encoded": 60, + "decoded": 60, + "fps": 29.9851, + "kbps": 1934.19, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 3.8464, + "firstReducedMs": -1 + }, + "b": { + "encoded": 0, + "decoded": 0, + "fps": 0, + "kbps": 0, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": -1, + "firstReducedMs": -1 + }, + "aEncodeMs": 3, + "bEncodeMs": 2, + "aEncodeUsage": 23, + "bEncodeUsage": 24 + }, + { + "integrity": "pass", + "pooled": true, + "adjuster": false, + "groupAdjuster": true, + "unsafeSkip": false, + "encodes": 339, + "hits": 260, + "splits": 5, + "rateSplits": 1, + "dependencySplits": 2, + "created": 5, + "joins": 3, + "live": 0, + "maxLive": 2, + "callbacks": 599, + "samePayloadCompared": 260, + "decodedHashMismatches": 0, + "aWantsUpdates": 3, + "bWantsUpdates": 3, + "aMinPixels": 230400, + "bMinPixels": 230400 + } + ], + "raw-unsafe": [ + { + "phase": "healthy", + "seconds": 6.00021, + "encodes": 187, + "hits": 173, + "a": { + "encoded": 180, + "decoded": 180, + "fps": 29.999, + "kbps": 1778.23, + "errors": 0, + "keyframes": 2, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 7.4999, + "firstReducedMs": -1 + }, + "b": { + "encoded": 180, + "decoded": 180, + "fps": 29.999, + "kbps": 1781.73, + "errors": 0, + "keyframes": 3, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 12.4981, + "firstReducedMs": -1 + }, + "aEncodeMs": 4, + "bEncodeMs": 2, + "aEncodeUsage": 37, + "bEncodeUsage": 34 + }, + { + "phase": "skipped-input", + "seconds": 3.00024, + "encodes": 96, + "hits": 83, + "a": { + "encoded": 90, + "decoded": 90, + "fps": 29.9976, + "kbps": 1877.57, + "errors": 0, + "keyframes": 1, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 3.1385, + "firstReducedMs": -1 + }, + "b": { + "encoded": 89, + "decoded": 89, + "fps": 29.6643, + "kbps": 1832.6, + "errors": 0, + "keyframes": 2, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 4.7292, + "firstReducedMs": -1 + }, + "aEncodeMs": 4, + "bEncodeMs": 1, + "aEncodeUsage": 27, + "bEncodeUsage": 24 + }, + { + "phase": "retired-b", + "seconds": 2.00145, + "encodes": 60, + "hits": 0, + "a": { + "encoded": 60, + "decoded": 60, + "fps": 29.9783, + "kbps": 1893.96, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": 3.9411, + "firstReducedMs": -1 + }, + "b": { + "encoded": 0, + "decoded": 0, + "fps": 0, + "kbps": 0, + "errors": 0, + "keyframes": 0, + "width": 640, + "height": 360, + "minWidth": 640, + "minHeight": 360, + "firstFullMs": -1, + "firstReducedMs": -1 + }, + "aEncodeMs": 3, + "bEncodeMs": 1, + "aEncodeUsage": 23, + "bEncodeUsage": 24 + }, + { + "integrity": "fail", + "pooled": true, + "adjuster": false, + "groupAdjuster": true, + "unsafeSkip": true, + "encodes": 343, + "hits": 256, + "splits": 5, + "rateSplits": 2, + "dependencySplits": 0, + "created": 4, + "joins": 3, + "live": 0, + "maxLive": 2, + "callbacks": 599, + "samePayloadCompared": 256, + "decodedHashMismatches": 29, + "aWantsUpdates": 3, + "bWantsUpdates": 3, + "aMinPixels": 230400, + "bMinPixels": 230400 + } + ] + } +} diff --git a/docs/research/data/webrtc-h264-pipeline.json b/docs/research/data/webrtc-h264-pipeline.json new file mode 100644 index 0000000..f3e66ec --- /dev/null +++ b/docs/research/data/webrtc-h264-pipeline.json @@ -0,0 +1,44 @@ +{ + "date": "2026-09-08", + "webrtcRevision": "6f37672d358475cd17544121a12494da454d85fb", + "sdk": "shiguredo m152.7977.0.2", + "chrome": "152.0.7977.82", + "scope": "Synthetic I420 -> existing Windows MFT H264 adapter -> one complete VSE -> two callbacks; exported bitstream decoded separately in Chrome", + "input": { "width": 640, "height": 360, "fps": 30, "budgetBps": 2000000 }, + "healthy": { + "seconds": 6.00065, + "physicalEncodes": 174, + "encodedPerConsumer": [174, 174], + "samePayloadComparisons": 174, + "firstOutputMs": 274.88, + "width": 640, + "height": 360, + "averageOutputKbps": 695.472 + }, + "retiredB": { + "seconds": 2.00067, + "physicalEncodes": 60, + "encodedPerConsumer": [60, 0] + }, + "lifetime": { + "physicalEncodes": 234, + "created": 1, + "maximumLive": 1, + "remainingLive": 0, + "splits": 0, + "reportedErrors": 0 + }, + "browserDecode": { + "codec": "avc1.42c033", + "exportedFrames": 180, + "decodedPerDecoder": [180, 180], + "dimensionsAndTimestamps": "pass", + "scope": "Both decoders receive the same exported input, not separate network deliveries" + }, + "notEstablished": [ + "weak-child budget grouping or independent adaptation", + "hardware overload adaptation or game/capture throughput", + "Pion transport, packet loss or Browser WebRTC jitter-buffer behavior", + "production VSE integration, package or deployment acceptance" + ] +} diff --git a/docs/research/embedded-media.md b/docs/research/embedded-media.md new file mode 100644 index 0000000..189eb3c --- /dev/null +++ b/docs/research/embedded-media.md @@ -0,0 +1,112 @@ +# Embedded Media Evaluation + +Reviewed: 2026-09-07. Status: evaluation, not a deployed transport decision. +Baseline: `3da8ecd22a3499e660f498e527ad4ffbaef08c82`. + +## Goal + +The owner requests source/library-level STUN and SFU inside the Screener Go +process. Bundled coturn/LiveKit executables, automatic installers, subprocess +supervision and embedding the full LiveKit service do not meet this goal. +The owner has also agreed to [node-local adaptive reuse](./node-local-adaptation.md): +each parent derives missing lower outputs for its direct children and reuses +them locally. That file owns the agreed model and unresolved encoding policy. +nginx/TLS replacement and capture-picker changes are outside this experiment. +The existing release/configuration candidate remains separate and untouched. + +## Invariants + +Keep one room authority and the P2P-first, single-graph, single-operation router. +The fallback retains one Host publication feeding server-side fanout to many +Viewers. Framework-managed simulcast may use bounded representations within +that publication; Viewer count must not add Host encoders or independent Host +uploads. Ordinary TURN forwarding of one Host connection per Viewer does not +meet this requirement and is not an SFU replacement candidate. +Retain H264/VP8, Opus, per-subscriber adaptation, retransmission, recovery and +the existing three STUN destinations. Do not trade weak-Viewer experience for a +smaller package, invent an application quality score or add a media ladder. +The media library, not Screener, must own congestion and layer decisions. +No recording, mixing, TURN relay, public port scanning or production test. +Selective local transcoding is now in research scope under the agreed model; +the existing fanout-only probe does not establish that capability. + +## Evaluation Order + +1. Audit library APIs, license, dependencies and H264/simulcast feedback paths. +2. Implement a small in-process STUN owner using Pion. Check Binding responses, + no relay allocation, occupied-port rollback and complete socket shutdown. +3. Exercise one publisher and two receivers through an SFU library on loopback. + Check fixed Host publication count, actual media, clean teardown and the library's adaptation ownership; + frame forwarding alone is not acceptance of a LiveKit replacement. +4. Compare added dependencies and binary size, then remove unnecessary glue. + A production adapter/wire cutover needs a decision supported by these results. + +No new production dependency or alternate runtime transport is accepted merely +because its example compiles. Keep test executables under the worktree's stable +`build/` path and avoid native GPU capture probes. + +## Initial Candidates + +- Pion STUN/TURN is already in the dependency graph and explicitly supports + embedding. Only STUN is needed; TURN allocations must remain unavailable. +- ion-sfu is library-oriented, but its inspected module still uses Pion WebRTC + v3 and ICE v2. Do not introduce another older media stack by default. +- inlivedev/sfu uses Pion WebRTC v4 and exposes room/client APIs. Its README + explicitly warns that its API is early-stage; listed features are claims to + verify, not proof of Screener's required behavior. +- Galene is useful architectural reference, not yet an accepted library adapter. + +## Current Evidence + +- The 2026-09-07 local STUN check passed three independent Binding responses, + zero TURN allocations, port release on shutdown and occupied-port rollback. + The adapter uses Pion's existing STUN-only server, not a new STUN parser. +- `inlivedev/sfu` is pinned for evaluation at + `v0.0.0-20260307060053-14b2448a8982` (MIT). A loopback Pion publisher with one + H264 RTP sender fed two receivers, each receiving 12 RTP packets in the check. + No extra Host sender was created for the second receiver. The source was a + synthetic 2x2 H264 sample, so this is not Browser decode, game quality, latency, + resource-drain or adaptive-layer acceptance. +- The candidate is imported only by the probe test. There is no production + adapter or signaling change. Its dependency graph also adds TURN v4 alongside + the existing v5 and upgrades `x/net` from 0.50.0 to 0.51.0; evaluate that cost + before selecting it. No fork or patch of the library has been introduced. +- Candidate source contains Pion GCC, RTCP feedback and simulcast switching. + It also requires publisher-layer/bitrate agreement and has application-oriented + defaults. A demand-driven publisher contract equivalent to our current behavior + has not yet been established. README feature lists alone do not close this gap. + +Reproduction uses `go test -c -o build/embedded-media/.test.exe` for +`./internal/server/stun` and `./internal/server/sfuprobe`, then executes each +binary with `-test.v -test.timeout=15s`. Quote the dotted flags in PowerShell. + +## Transport Direction + +One-to-many distribution is not tied to the LiveKit implementation. Keep WebRTC +as the primary Browser media path while evaluating a lighter embedded engine. +RTMP/RTSP/SRT distribution can also fan out one input, but a Browser-facing +gateway/player pipeline and equivalent media adaptation would still be needed. +Do not add that second transport merely to change the service's name. + +WHIP is HTTP signaling for WebRTC ingest, not a replacement media transport. +Its no-renegotiation constraint must be checked against live audio/source changes; +it is not adopted merely because the request/response looks simpler. The current +Screener signaling owner remains authoritative during this evaluation. + +## Sources + +- [Pion embeddable STUN/TURN](https://github.com/pion/turn) +- [ion-sfu module](https://github.com/ionorg/ion-sfu/blob/master/go.mod) +- [inLive SFU API and caveats](https://github.com/inlivedev/sfu) +- [inLive dependency contract](https://github.com/inlivedev/sfu/blob/main/go.mod) +- [Galene architecture](https://galene.org/README.html) +- [Browser-facing MediaMTX playback](https://mediamtx.org/docs/read/web-browsers) +- [SRT protocol](https://github.com/Haivision/srt) +- [WHIP media and signaling contract](https://www.rfc-editor.org/rfc/rfc9725.html) + +## 中文范围 + +本阶段是库级替换实验，不是把外部程序藏进安装包。先核实成熟模块能否保持 +现有观影体验，再决定接入。仅转发成功不代表自动降级、重传和恢复已验收； +必须保留一次 Host 发布、服务端多观众分发；不能用逐观众 TURN 上行替代。 +尚未修改生产路由、协议或部署。 diff --git a/docs/research/native-client-lifecycle.md b/docs/research/native-client-lifecycle.md index ea3a465..60d7f9e 100644 --- a/docs/research/native-client-lifecycle.md +++ b/docs/research/native-client-lifecycle.md @@ -1,10 +1,11 @@ # Native Client Lifecycle -- Reviewed: 2026-09-05 +- Reviewed: 2026-09-09 - Scope: Windows capture idle semantics, native loopback input isolation, and Browser visibility of an unexpected Client disconnect. -- Status: static-source, preview-queue and Client-crash checks pass; Windows 10 - monitor startup and game-specific window replacement remain unresolved. +- Status: quiet-source, preview-queue and Client-crash checks pass. The owner + confirmed Windows 10 monitor sharing resolved; game-specific window replacement + is not established by the quiet-source checks. ## Current Evidence @@ -40,7 +41,7 @@ runtime change for either is claimed here. - Native Pion candidate input is syntax-checked with the existing ICE parser, bounded before queuing, and discarded when malformed, stale, or over capacity. Repeated answers for an edge are idempotent. Profile updates remain fenced to - that same share session in the current loopback v8 wire. + that same share session in the current loopback v9 wire. - An unexpected Browser control-socket close now notifies the active Host owner; intentional user cleanup remains silent and uses the existing share fence. @@ -61,6 +62,27 @@ runtime change for either is claimed here. minimized, then produced 17 frames within 540 ms of restoration. All owned processes, ports and profiles were cleaned up. Neither check establishes the reported Windows 10 monitor-start failure or game HWND recreation as fixed. +- On 2026-09-08, the current multi-output Windows VP8 Host gate captured only + its own animated Chrome window. Live presets, paused changes, minimize/change + resolution/restore/another change, and source replacement preserved the + Viewer's media object. Closing the actual source ended the share; restarting + delivered 30 new Viewer frames. Server-accepted Native quality evidence and + process/port/profile cleanup passed. The gate's stale Node JSON-log assertion + was corrected to the current Go text format, without changing evidence + eligibility. This is not H264 overload, Win10, or game-window replacement + evidence. The same owned-window sequence subsequently passed H264. Further + physical runs use the serialized plan in [TODO](../todo.md). + +The multi-output Windows worker initially reapplied the source's frame-rate +limit from each worker's first input timestamp. With jitter and a later worker +start, this second limiter could reject frames already admitted by capture. +Captured inputs now carry their existing cadence bound; a worker limits only +when it needs a lower rate. Received encoded input has no such bound and keeps +its output limiter. No capture ceiling, media clock or quality setting changed. +In the same owned-window H264 720p15 SFU gate, the reported Native output went +from 12 fps before this correction to 15 fps after it. Both runs decoded 300 +720p frames, applied a live 480p change and cleaned up. This is a local cadence +comparison, not a claim about hardware overload or all games. ## Open Capture Reports @@ -72,6 +94,9 @@ request has been established. Community reports of disabled capture services or invalid display contexts are leads, not a diagnosis of this user's machine. Projected WinRT exceptions now reach the existing capture-error boundary instead of escaping `std::exception`; that correction alone does not prove startup fixed. +On 2026-09-09, the owner confirmed whole-display sharing works and closed this +report. That field confirmation does not establish which earlier correction or +environment change resolved it; no further workaround is justified by this report. The original converter always fitted the captured pixel aspect into the output: 1280x960 into 1920x1080 yielded 1440x1080 content and 240-pixel side bars, even @@ -95,6 +120,106 @@ replaces the underlying window. Reattaching to another window is not equivalent to resuming the same selected target; this remains separate from the verified quiet/minimized-source behavior. +The 2026-09-08 owner clarification names CS2, fullscreen 4:3 stretched: leave the +game, change advanced share settings, then return; the share ends on return. +The ordinary-window gate does not reproduce exclusive-fullscreen/display-mode +changes. Capture replacement overlap, target identity and device state remain +separate hypotheses until that exact transition is observed. + +Later that day, the Windows 11 candidate ran the actual installed CS2 in +fullscreen 1920x1440. The desktop changed from 2560x1440 while the game was +minimized to 1920x1440 after return. Two real HostPage pending updates returned +to the game after 1,703 ms and 1,545 ms: H264 1080p30 to 720p30 and back. +The Viewer received both actual sizes, retained its media object, and the game +kept its HWND/process generation. The current candidate did not reproduce the +reported interruption in this menu-level sequence. Game settings were restored +byte-for-byte and all owned processes, ports and Browser profiles closed. +An additional prolonged-background case failed to keep CS2 minimized before +the settings action, so it is not a passed or failed capture check. Vendor-side +stretch fidelity, in-match execution and the affected machine remain unverified. + +The end-to-end review did confirm a separate presentation defect: share +termination during a pending update cleared its token but retained the advanced +draft. A restart used the last applied settings while controls could show the +failed draft. Cleanup now restores the applied settings through the existing +commit helper; no new state or recovery mechanism was added. This alone is not +claimed as the cause of CS2 capture termination. + +The later owner-supplied `cbdd751` CS2 diagnostic bundle records three 720p30 +update requests, all rejected while the existing 1080p30 share remains live. +It contains no successful 720p commit or explicit rollback-to-1080 request. +The Browser's failed-request rollback explains its controls returning to the +committed profile, but the original generic error does not identify which +preparation step failed or explain the reported brief Viewer-size change. +Current diagnostics add fixed preparation rejection stages without changing +capture recovery policy. Do not label the affected-device report resolved. + +The matched `0675265` local reproduction then held CS2 minimized continuously +for 8.6 seconds during a 720p request. Preparation failed with `wait-timeout` +after 5,015 ms and controls reverted; the Viewer retained 1080p and its media +object. The identical request succeeded in 438 ms after restoring the game. +The affected machine's subsequent bundle independently records eleven update +timeouts, three successes and one cancellation caused by share termination. +This establishes a quiet-source/readiness error in the live-update path, not +an old setting request overwriting a successfully installed replacement. + +On 2026-09-09, matched `cbb3d77` passed the corrected path through actual Windows +Graphics Capture and Browser playback. An owned source stopped repainting and +remained minimized for 8,378 ms: the 720p request stayed pending while the Viewer +retained 1080p. Restoring the source delivered 720p in 1,868 ms; the next 1080p +request delivered in 480 ms, retaining the media object. Both updates succeeded; +all processes, ports and Browser profiles closed. This closes the reproduced +quiet-source timeout defect. CS2 foreground automation did not complete that +same post-fix sequence, so this does not establish game-specific display-mode +or HWND replacement behavior. + +That same bundle also contains two independent `adaptive-output` capture +failures at H264 1440p60, one without a settings update in progress. A controlled +1440p60/12 Mbps high-entropy input produced a valid 1,609,216-byte keyframe and +reproduced the one-MiB capture-envelope rejection. The MFT and RTP access-unit +owners already permit four MiB. Aligning capture readers/writers removes that +false fatal rejection, but does not by itself establish delivery: the separate +500-packet forwarding/pacing bound also loses that large recovery frame. Both +boundaries must be verified before describing the large-frame report as fixed. + +The aligned 4,096-packet source/pacing/reassembly window subsequently delivered +two copies of that actual keyframe byte-for-byte with zero paced drops or +assembly loss. The 500-packet comparison dropped 1,182 packets and reconstructed +none. Active fixture heap usage increased by roughly ten to thirteen MiB; +LiveKit's packet cache still grows lazily and all queues remain bounded. Go +capture and transport now share the encoded-frame bounds owner. This fixes a +reproduced legal-input failure, not proof that every compound `adaptive-output` +error in the field had the same cause; distinct failure stages preserve that +remaining distinction. + +The `cbdd751` Vivaldi bundle shows two successful Native H264 1080p30 capture +starts followed by a local outbound PeerConnection remaining connecting for +about 7.985 seconds. The Browser bridge's eight-second startup timeout then +stops the Native share cleanly. This is a local media-bridge failure boundary, +not evidence that Windows capture or the control WebSocket failed. That bundle +does not distinguish ICE from DTLS or a Browser policy. Current diagnostics +record those states and candidate-type counts separately; the UI reports a +connection error. The subsequent `0675265` bundle shows four successful capture +starts, each with two Native UDP host candidates, no usable remote candidates, +ICE stuck checking and no DTLS connection before the same deadline. + +The owner then confirmed the affected Vivaldi report was resolved: a VPN +extension forcibly disabled **Broadcast IP for Best WebRTC Performance**; +correcting the setting/extension restored Native sharing. The confirmed cause +is that extension's WebRTC policy, not unsupported Windows capture. VPN +extensions may reapply the override; the +[Chromium WebRTC FAQ](../../cmd/screener-client/README.md#chromium-webrtc-connections) +owns the user steps. This does not authorize a Browser-specific transport fallback. + +An isolated Chrome 152.0.7977.82 data-channel A/B verifies the policy +mechanism: default handling gathers one UDP host candidate per peer and connects +ICE/DTLS in 33 ms; `disable_non_proxied_udp` gathers no candidates and remains +unconnected at 8,009 ms. The check uses fresh profiles and the effective +`--webrtc-ip-handling-policy` switch, without capture, encoding or user-profile +changes. This establishes Chromium's policy mechanism; the affected Vivaldi case +was resolved by the owner's setting correction above. Extra STUN or a longer +timeout cannot restore candidates disabled by that policy. + Primary references: [CreateForMonitor](https://learn.microsoft.com/en-us/windows/win32/api/windows.graphics.capture.interop/nf-windows-graphics-capture-interop-igraphicscaptureiteminterop-createformonitor), [CreateFreeThreaded](https://learn.microsoft.com/en-us/uwp/api/windows.graphics.capture.direct3d11captureframepool.createfreethreaded), [capture item closure](https://learn.microsoft.com/en-us/uwp/api/windows.graphics.capture.graphicscaptureitem.closed), diff --git a/docs/research/node-local-adaptation.md b/docs/research/node-local-adaptation.md new file mode 100644 index 0000000..e2a5562 --- /dev/null +++ b/docs/research/node-local-adaptation.md @@ -0,0 +1,217 @@ +# Node-Local Adaptive Encoded Reuse + +Reviewed: 2026-09-07. [ADR-0013](../adr/0013-embedded-node-local-media.md) now owns +the accepted model and highest-demand envelope. Codec backends and timing +values still need implementation evidence. ADR-0007 remains the runtime contract. +The [design proposal](./node-local-media-design.md) narrows implementation from +the measurements below; it does not reopen the agreed topology/reuse model. + +## Agreed Model + +ADR-0013 is the sole owner of the accepted direct-child model and policy. This +file retains evidence and alternative comparisons, not a second design contract. +The implementation design is owned by [node-local media design](./node-local-media-design.md). +`base/native` names the selected source quality; it is not a second output +alongside an identical `high` encoding. + +## Two Workstreams + +- Self-contained Screener Server: library-level STUN/media components inside the + executable, not bundled external service processes or an automatic installer. +- Adaptive reuse: local derivation, shared lower outputs, independent child + delivery and quality recovery, usable by Native Client and the server. + +These share media-source and lifetime concepts but need separate acceptance. +A forwarding-only SFU library does not provide the second workstream by itself; +server-side derivation needs real decoding/encoding capacity. Pure Browser +peers can consume normal WebRTC outputs and benefit from capable upstreams, +but have no generic cross-PeerConnection encoded-forwarding API. + +## Existing Building Blocks And Gaps + +- `mediaedge.Source` already forwards encoded RTP to multiple Pion edges. +- Native GCC observations currently drive route evidence, not the shared + encoder. A sustained weak edge may use a measured Browser-sender exception. + That exception is not yet a reusable derived-encoding pool. +- Existing capture encoders, generation fences, keyframe requests and bounded + route operations are reusable. A new standalone quality controller, global + node score or periodic rebalancer is not assumed. +- Current routing reacts primarily to persistently degraded senders. A healthy + edge forwarding already-low input may not trigger a move. Transport health + and available source quality must be distinguished before claiming recovery. +- Simulcast, encoder allocation, active encoding, layer forwarding and bandwidth + probing are different operations. Pausing a representation must not be + mistaken for destroying its codec object or reopening capture. + +## Open Choices + +- Always encode all configured lower outputs, enable all after the first weak + child, or activate only the representations currently demanded. +- Keep unused encoder resources warm or release them; decide from cold-start, + first-keyframe, GPU-session, memory and idle-CPU measurements. +- Use a bounded representation set and one parameter owner per shared output. + Determine which mature component owns demand grouping and adaptation; do not + assume that independent child GCC loops can all mutate one shared encoder. +- Coordinate a usable-frame handoff before switching, preserve unaffected + branches, and avoid repeated downgrade/recovery churn. +- Check server software-transcode cost and platform-native hardware availability + independently. A network-capable parent is not necessarily a capable encoder. + +## Simulcast And Dynacast Findings + +These observations were checked against LiveKit Server 1.13.6 and JS SDK 2.22.1, +not inferred from the words "automatic" or "simulcast": + +- Simulcast advertises several encoded representations of one source. The SDK + constructs a finite encoding set before publishing; it does not create an + arbitrary new representation for every bandwidth sample or subscriber. +- The pinned screen-share defaults create the original plus one lower spatial + representation for ordinary screen sizes, not invariably three encoders. +- The server's Dynacast video manager aggregates the highest subscribed quality + across subscribers/downstream SFU nodes. Its update enables existing qualities + at or below that maximum. Thus a high requirement can retain lower fallback + representations; this is not exact per-current-layer consumer reference + counting. Do not claim every unused individual lower layer is always stopped. +- A higher quality request is not subject to downgrade-only debounce. Reduced + maximum demand is debounced; the pinned server default is five seconds. This + is an upstream implementation fact, not a proposed Screener timing constant. + Publisher deactivation debounce is not a five-second delay before reducing + traffic to a congested subscriber. +- The SDK applies the selected active flags through `RTCRtpSender.setParameters` + on the existing sender. WebRTC's `active=false` preserves SSRC identity. It is + not a guarantee that every Browser releases, or retains, hardware encoder + resources in the same way. +- Subscriber forwarding choice is separate from publishing activity and from + destroying an encoder. A decoder-safe layer switch needs an appropriate + random-access point and consistent RTP/timestamps, not just another byte array. +- LiveKit's prober documents padding-based capacity probes independently of + video-layer production. Higher-video-layer probing is another possible + strategy, but cannot be substituted for a calibrated congestion controller. + +The useful lessons are finite representations, distinct demand/transport +ownership, asymmetric activation/deactivation and independent network probing. +They do not imply importing the whole LiveKit service or extending its +subscriber-tree demand aggregation to override our direct-child derivation rule. + +## Strategy Comparison + +Shared encoding and per-child selective delivery are common to both compared +policies. The owner explicitly excludes Native-plus-Browser and repeated full +encoders as the next experiment's reference baseline. Compare exact demanded +outputs against the maximum-demand envelope, not shared versus unshared media. + +| Strategy | Main benefit | Cost to verify | +| --- | --- | --- | +| Base plus all lower encodings always active | Lower outputs are already being produced when needed | Permanent encode/GPU cost even for an entirely healthy tree; still needs a switch point | +| Base only, then all lower outputs after any weak child | Small policy, some idle savings | First downgrade remains a cold start; one weak child starts outputs nobody requested | +| Finite outputs activated only as needed | Avoids unused representations and shares compatible demands | Cold start and churn if codecs are constantly destroyed/recreated | +| Demand activation with bounded warm retention or a minimal fallback output | Potential compromise between steady cost and transition delay | Requires measurements to justify retention or pre-encoding; not free memory/session capacity | + +The owner's 2026-09-07 preference makes the LiveKit-style maximum-demand +envelope a first-class candidate: keep the highest requested representation and +its lower fallbacks active; stop unused higher outputs when all direct children +need less. Compare it with exact demand activation, not with an assumption that +exact demand is inherently better. Prefer the simpler envelope if its measured +cost is small and switching is better. Good network capacity alone does not +prove spare encoding capacity or cheap lower outputs. + +Both policies retain the original source authority and the ability to recover; +stopping an unneeded high output does not lower capture or discard the best +received input. Exact demand can retain inactive codec resources for a bounded +comparison instead of repeatedly destroying them. No production timing value, +extra polling loop or prediction score has been selected. +Do not blindly create a fallback encoder set at every forwarding relay: that +would add decode/encode work to the entirely healthy tree. Reusing received +encodings and deciding how an already-needed encoder set stays active are +separate decisions; publisher measurements cannot stand in for relay costs. +Where the media framework supports it, negotiate the finite representation +capability up front and activate existing identities; a quality change should +not reopen capture or recreate a PeerConnection merely to add an encoder. + +Preparing/retaining an encoder, actively encoding frames and forwarding packets +are separate costs. Share stop, source replacement and failed ownership still +retire their resources immediately; warm retention cannot revive a retired source. + +## Reusable Components + +| Component | Useful capability | Boundary | +| --- | --- | --- | +| Existing `mediaedge.Source` and Pion RTP/WebRTC/interceptors | Encoded fanout, per-edge transport and feedback | No decoder/transcoder or shared-variant policy is supplied automatically | +| Existing Native encoders and generation/keyframe ownership | Platform encode paths already integrated with Screener | Received-source decode/scale and shared derived outputs still need design | +| LiveKit Dynacast, layer selector and prober sources | Reference for demand aggregation, safe switching and recovery probes | Apache-2.0 code; inspect imports/ownership before any extraction, not whole-service embedding | +| libvpx multi-resolution VP8 API/example | Multiple spatial encodes through one coordinated API; libvpx is already used | VP8-specific; no promise that H264 hardware sessions have the same cost curve | +| OpenH264 AVC simulcast API | Library H264 encode/decode and per-spatial-layer configuration | Software codec; inspect realtime cost and actual source/binary distribution terms before selection | +| FFmpeg `libavcodec` and frame/filter APIs | Embedded decoding, scaling and hardware/software encoding | A library option, not launching `ffmpeg`; build/license/size cost remains to compare | +| Pion mediadevices | Go codec-controller and raw-frame broadcast interface reference | Cgo/native codecs still exist; not a reason to replace the working capture adapters | +| GStreamer `tee`/queues and codec elements | Shared decode/frame flow and isolated output branches | `webrtcsink` currently encodes per consumer, so it does not directly solve shared derived output | + +No candidate has yet demonstrated the complete local derivation, reuse and +recovery contract. The current inLive probe proves forwarding only. A small SFU +library does not by itself replace a transcoder. Do not add FFmpeg/GStreamer or +change the current Native codec stack on documentation claims alone. + +## Focused Comparison Plan + +Use the same source, codec, frame rate and direct-child count for each arm: + +1. Healthy children: verify no derived encode is added just for fanout. +2. One weak child, then two compatible weak children: verify one derived output + is sufficient for both and the original branch stays unchanged. +3. Bandwidth drop/recovery and short oscillation: compare cold versus retained + encoder startup, first usable frame, freezes, CPU/GPU, memory and actual RTP + traffic. Count extra decoders and CPU/GPU frame copies as well as encoders; + Browser-decoded preview frames are not automatically zero-copy Native input. + Measure extra IDR/probe traffic, not only the selected bitrate. +4. Low inherited input on an otherwise healthy edge: distinguish lack of better + source from lack of link capacity; exercise higher-input recovery or the + existing bounded route operation without a new global optimization loop. +5. Source replacement, child departure and stop: verify generations and release + of decoder/encoder surfaces, tracks and queues. + +The first [CPU-only encoder comparison](./node-local-encoding-probe.md) is complete. +It compares existing Native VP8 encodes and scripted direct-child demand, not +network adaptation or playback. It supports keeping the maximum-demand envelope +as a candidate, but also exposes timestamp-sensitive resume behavior in a naive +warm-encoder composition. Validate mature multi-resolution pause/resume semantics +before implementing retention. H264 hardware, relay decode/scale and actual +receiver switching remain open; no production policy changed. +The subsequent [software-preference decode comparison](./node-local-browser-probe.md) +now includes VP8/H264, scaling and two real VideoDecoders, plus received-source +relay derivation. It measures local component readiness only, not WebRTC delivery +or congestion. The design treats representation bookkeeping as small and puts +the remaining overhead work in codec lifecycle, shared decode/surfaces and the +transport library's switching/probing path. + +## Primary References + +- [LiveKit JS encoding construction, 2.22.1](https://github.com/livekit/client-sdk-js/blob/v2.22.1/src/room/participant/publishUtils.ts) +- [LiveKit JS publishing layer activation, 2.22.1](https://github.com/livekit/client-sdk-js/blob/v2.22.1/src/room/track/LocalVideoTrack.ts) +- [LiveKit Dynacast quality aggregation, 1.13.6](https://github.com/livekit/livekit/blob/v1.13.6/pkg/rtc/dynacast/dynacastqualityvideo.go) +- [LiveKit activation/debounce, 1.13.6](https://github.com/livekit/livekit/blob/v1.13.6/pkg/rtc/dynacast/dynacastmanagervideo.go) +- [LiveKit default pause delay, 1.13.6](https://github.com/livekit/livekit/blob/v1.13.6/pkg/config/config.go) +- [LiveKit capacity probing design, 1.13.6](https://github.com/livekit/livekit/blob/v1.13.6/pkg/sfu/ccutils/prober.go) +- [WebRTC active encoding semantics](https://www.w3.org/TR/webrtc/#dom-rtcrtpencodingparameters-active) +- [libvpx multi-resolution example](https://github.com/webmproject/libvpx/blob/main/examples/vp8_multi_resolution_encoder.c) +- [libvpx multi-encoder API](https://chromium.googlesource.com/webm/libvpx/+/master/vpx/vpx_encoder.h) +- [OpenH264 encoder API](https://github.com/cisco/openh264/blob/master/codec/api/wels/codec_app_def.h) +- [OpenH264 AVC simulcast release history](https://github.com/cisco/openh264/blob/master/RELEASES) +- [FFmpeg codec send/receive API](https://ffmpeg.org/doxygen/trunk/group__lavc__encdec.html) +- [Pion mediadevices codec boundary](https://github.com/pion/mediadevices) +- [GStreamer branch isolation](https://gstreamer.freedesktop.org/documentation/coreelements/tee.html) +- [GStreamer WebRTC consumer encoding](https://gstreamer.freedesktop.org/documentation/rswebrtc/index.html) + +## 中文确认 + +核心是“只负责直属下游”：有合适编码就转发，缺少低档才由直属上级按需派生， +相同需求复用派生输出。不是 Host 为全树预编码所有档位，也不是每条弱边独立 +编码。Host 同样遵守此规则。基准档、常用档的名字不能造成重复编码。 + +拓扑优化负责修复上游输入质量受限，局部媒体模块负责可用表示的派生、复用和 +选择。稳定传输低画质不等于路径已经最优；现有优化不能直接视为已覆盖这种 +恢复。当前先研究按需/预编码策略及成熟实现，尚未改动生产媒体或路由。 + +优先把 LiveKit 式“最高需求档及以下保持编码”与精确按需启用作为平等候选， +不预设后者更优；若常备低档成本小、切换更好，就选择更简单的策略。所有直属 +下游都不需要高档时可以停掉高档输出，但不修改捕获或丢弃最好的输入来源。 +先用已有 VP8 编码器短时测量成本和关键帧启停，不把结果外推成 H264 硬件或 +真实网络验收。具体库、档位与启停时间仍未作为生产决策。 diff --git a/docs/research/node-local-browser-probe.md b/docs/research/node-local-browser-probe.md new file mode 100644 index 0000000..1ed1257 --- /dev/null +++ b/docs/research/node-local-browser-probe.md @@ -0,0 +1,106 @@ +# Selective Encoding And Decode Probe + +Measured: 2026-09-07. Both arms share each encoded output and deliver only the +representation selected by each direct child. This extends the +[Native VP8 cost probe](./node-local-encoding-probe.md), not the production UI. +[Raw component measurements](./data/node-local-browser.json) include all 16 runs +and the exact worker SHA-256. + +## Method And Limits + +- Isolated Chrome 152.0.7977.82 on Windows, WebCodecs with `prefer-software`, + realtime latency and constant bitrate requests; VP8 and `avc1.420028` supported. + The API does not identify the actual backend here. This is not a Native MFT, + H264 hardware or Browser WebRTC sender benchmark. +- One deterministic moving NV12 input, 1080p30, passed to finite 1080p/540p/270p + encoder configurations at 5/1.25/0.3125 Mbps. The codec performs any required + scaling; there is no custom scaler. These remain measurement fixtures. +- Two VideoDecoders consume the selected encoded output. They decode the same + shared chunk when they request the same layer. Dimensions and timestamps are + checked for every delivered frame. There is no RTP, network, jitter buffer, + retransmission, packet pacing, A/V sync or pixel-quality comparison. +- Seven phases of 32 frames: `H/H, H/L, L/L, M/M, H/H, H/L, H/H`. Two rounds + reverse policy order, for Host and relay, with both codecs. Exact demand + destroys inactive codecs; envelope keeps all layers through the maximum + requested level active. This does not emulate LiveKit's demand/debounce loop. +- Relay receives an already encoded H stream prepared outside timing. H passes + directly to children. Only missing lower output activates one input decoder; + both arms decode 128 of 224 input frames, not once per child. No lower encoder + runs in relay H/H phases. An envelope relay with H/L also produces M as a + fallback; it never re-encodes the received H. +- Demand changes align with upstream keyframes. Receiver configuration changes + start on a keyframe. This is a best-case local switching fixture, not evidence + for arbitrary-phase PLI response or weak-network switching. +- Outputs reach their consumers independently; H forwarding does not wait for + lower encodes. The initial scaffold had an all-encoder delivery barrier; it + was removed before the recorded run. Each source iteration still waits for + all work to finish to bound the probe. Per-frame encoder flush simplifies the + fixture and may differ from a continuously queued production codec. +- Reported times are elapsed local readiness/work intervals, not total CPU + usage. Parallel encoders can hide additional CPU work behind similar elapsed + time. Use the Native probe for the separately measured CPU-cycle comparison. + +## Results + +Means across the two rounds; each consumer decoded all 224 expected frames in +every run, with the requested dimensions and timestamps. + +| Codec / role | Exact total processing ms/frame | Envelope total processing ms/frame | Exact / envelope encode-stage readiness ms | +| --- | ---: | ---: | ---: | +| VP8 Host | 6.85 | 6.93 | 2.83 / 2.92 | +| VP8 relay | 6.37 | 6.52 | 0.50 / 0.70 | +| H264 Host | 5.60 | 5.63 | 4.46 / 4.51 | +| H264 relay | 2.27 | 3.27 | 0.92 / 1.93 | + +The bookkeeping/configuration-call mean is below 0.02 ms/frame in every arm. +This only bounds the small finite-set implementation tested here; it does not +measure an elaborate adaptive scheduler or all asynchronous codec setup costs. +The extra middle output in relay H/L makes the envelope do more work even though +the child never receives those packets. Across each relay run, exact encoded +96/32/0 L/M/H frames and envelope 128/96/0; H was reused in both. + +Host's first H-to-L switch took 4.9-5.4 ms exact versus 4.2-4.3 ms envelope for +VP8, and 3.0-3.2 ms versus 2.2-2.5 ms for H264. That small local difference does +not prove a noticeable playback benefit. + +H264 Host's later return to H measured 192-195 ms exact versus 36-38 ms envelope, +repeating with reversed order. Both policies had stopped H during all-low and +middle phases, so this is not simply "H was already encoding". Backend resource +and scheduling effects have not been separated. Record it as an observed +implementation-dependent difference, not an inherent guarantee of the policy. +Both H264 policies also had roughly 90-100 ms middle-layer cold transitions. +An envelope does not eliminate every cold-start delay. + +Output reuse did not imply broadcasting every layer. For example, VP8 relay +generated 260,544 versus 583,797 derived bytes, while both policies selected +exactly 4,946,988 total bytes for their two receivers. These are codec payload +counts, not measured network throughput. Extra encoded bytes were unused. + +## Implications + +The expensive parts are codec work, cold activation and received-frame +decode/scale, not a three-entry demand set. Common finite representations can +support either policy without a second pipeline. The results do not establish +that exact demand always has high overhead, or that always-active layers are +free. Prefer a small framework-style owner and avoid a custom warm resource +manager until evidence requires one. + +The [design proposal](./node-local-media-design.md) takes the maximum-demand +envelope as the first integration candidate for an already-needed encoder group, +while preserving zero-transcode relay paths and leaving hardware/network +acceptance explicit. No source runtime, wire, dependency or production change +was made by this probe. + +## Reproduction + +```text +node scripts/encoded-variants-browser.mjs +``` + +Open the printed loopback URL in an isolated Chrome and activate its one button. +The bounded worker posts its structured result to +`build/embedded-media/browser-variants-result.json`. Close the browser and stop +the probe server afterward. This is opt-in research, not a new default CI job. +Input is synthetic; it captures no user screen or private media. + +Reference: [WebCodecs processing and resource model](https://www.w3.org/TR/webcodecs/). diff --git a/docs/research/node-local-encoding-probe.md b/docs/research/node-local-encoding-probe.md new file mode 100644 index 0000000..3db8ca4 --- /dev/null +++ b/docs/research/node-local-encoding-probe.md @@ -0,0 +1,124 @@ +# Node-Local Encoding Cost Probe + +Measured: 2026-09-07. This is a component experiment, not a media-policy decision. +[Node-local adaptive reuse](./node-local-adaptation.md) owns the design model. +[Full measurement data](./data/node-local-vp8.json) includes both rounds, phase +counts, timings, keyframe sizes and measured source digests. + +## Method + +- Windows 11 build 26200, Ryzen 7 9700X, 16 logical CPUs; existing libvpx 1.17.0 + and unmodified `Vp8Encoder`, four threads per encoder, realtime CBR, no GPU. +- Synthetic NV12 texture at 30 fps: mostly static with a small changing patch, + or scrolling detail. Each size uses the same deterministic scene. Input + preparation, scaling, decoding, capture and network are excluded. +- Fixture outputs are 1920x1080 at 5 Mbps, 960x540 at 1.25 Mbps and 480x270 at + 0.312 Mbps. These half/quarter-size fixtures are not a production ladder. +- Steady arms run four seconds, excluding the first second. Serial scheduling + isolates encoding work; two passes reverse scenario order. This is not a + comparison against actual Browser WebRTC senders or hardware simulcast. +- Trace arms replay two direct children for two seconds per phase: + `H/H -> H/L -> L/L -> M/M -> H/H -> H/L -> H/H`. A new requested output needs + an encoded keyframe, even if that layer was already being encoded. +- `envelope` encodes all outputs at or below the maximum scripted demand. + `exact-cold` encodes only demanded outputs and destroys inactive codecs. + `exact-warm` retains at most three codec objects through the 14-second trace, + without encoding inactive outputs. No LiveKit BWE or debounce is emulated. +- Elapsed work uses a monotonic clock. Whole-window process CPU cycles include + worker activity. `GetProcessTimes` is also retained but varied considerably in + these short windows; conclusions below use cycles and elapsed work, not its + apparent CPU percentage. Resident-memory deltas are not leak/peak-memory proof. + +## Steady Results + +Means across two rounds. CPU work is relative process cycles per source frame, +normalized to one high encode within the same scene, not whole-machine usage. + +| Active encodings | Mostly static ms/frame | Motion ms/frame | Relative CPU work, static / motion | +| --- | ---: | ---: | ---: | +| High | 4.26 | 6.52 | 1.00 / 1.00 | +| Two independent high encodes | 8.47 | 12.93 | 1.95 / 1.95 | +| High plus low | 4.68 | 7.11 | 1.09 / 1.09 | +| High, middle and low | 5.84 | 8.71 | 1.35 / 1.32 | +| Middle plus low | 1.69 | 2.35 | 0.39 / 0.36 | +| Low | 0.45 | 0.65 | 0.09 / 0.10 | + +Three outputs cost about 31-32% fewer CPU cycles than two full-size outputs in +this fixture, but 32-35% more than one. Lower outputs are not free. The three +sizes contain 1.3125 times the high output's pixels, so this does not establish +special multi-resolution work sharing: the current library build has +`CONFIG_MULTI_RES_ENCODING=0` and uses separate encoder contexts. + +Steady high output produced 90 measured frames and the same total output bytes +across the high-containing arms. This is a count/size check, not a bitwise or +perceptual-quality proof. Two-high motion had one work sample over 33.3 ms in +each round; the other steady arms had none. This is not a Viewer freeze count. +Motion resident-memory deltas were about 70 MiB for high, 93 MiB for three outputs +and 138 MiB for two high encodes, excluding the prebuilt input allocation. + +## Demand And Resume + +The envelope used about 26% more CPU cycles than exact-cold on the scripted +trace. The extra output counts were visible: low/middle/high encoded +420/360/300 frames versus 180/60/300 for exact demand. Two children requesting +the same low output did not double its encode count. Stopping high output in +the all-low phase did not discard the input fixture. + +Naive warm retention saved only about 5 ms of total codec initialization across +14 seconds, compared with cold recreation, and retained about 23 MiB more +resident memory at trace end. It also changed resumed output substantially: + +| High resumes at frame 240 | Keyframe bytes, both rounds identical | Ready time, two rounds | +| --- | ---: | --- | +| Fresh codec after pause | 32,268 | 22.79 / 19.76 ms | +| Retained codec, original source timestamps | 228,469 | 21.79 / 21.56 ms | +| Retained codec, timestamp-gap ablation | 31,079 | 13.24 / 12.90 ms | + +The ablation changes only the encoder PTS to count active encoded frames while +retaining the same input pictures. It isolates the timestamp-gap effect; it is +not an accepted replacement for source timestamps, RTP timing or A/V sync. +No network was measured, so the larger keyframe is evidence of burst risk, not +a demonstrated network stall. + +The pinned libvpx source derives an internal frame rate from input timestamps. +Its multi-resolution path explicitly uses the lowest active layer's frame rate +to avoid a paused upper layer misreading the timestamp gap. That is a mature +reuse lead, not a reason to implement an application clock workaround. The +current probe did not enable or validate that multi-resolution implementation. + +## Decision And Next Evidence + +Keep the maximum-demand envelope as a serious, simpler candidate. This result +does not prove that exact demand is worth a custom retention controller, nor +that the envelope switches receivers faster. Serial high-first scheduling and +keyframe work still affect its local readiness time. Do not add a warm pool or +timing workaround to production from this experiment. + +Next compare the library's coordinated multi-resolution behavior, then measure +H264 hardware cost and actual decode/forward switching under bounded congestion. +Relay decode, scaling, surface copies, output reuse and inherited-low-input +recovery are separate remaining costs. Blindly starting lower encoders at every +healthy forwarding relay would violate the one-encode reuse objective. + +## Reproduction + +Use the existing pinned libvpx build produced by the Windows Client build. From +a Visual Studio x64 developer shell, with `VPX_INCLUDE` and `VPX_LIBRARY` pointing +to that build: + +```bat +cl /nologo /std:c++20 /EHsc /O2 /MT /W4 /WX /external:I "%VPX_INCLUDE%" /external:W0 native/capture/windows/encoded_variants.probe.cpp native/capture/windows/vp8_encoder.cpp /Fo:build/embedded-media/ /Fe:build/embedded-media/encoded-variants.probe.exe "%VPX_LIBRARY%" /link /LTCG psapi.lib +node scripts/encoded-variants-probe.mjs +``` + +Create the `build/embedded-media` directory first. The Node entry runs 32 bounded +processes sequentially with frame/count checks and writes its report under that +directory. No default CI, application dependency, capture, room or production +configuration was changed. All generated executables stay in this stable build +path. The clock-ablation arm exists only to reproduce the observed pause effect. + +## Primary Sources + +- [Pinned libvpx timestamp and multi-resolution handling](https://github.com/webmproject/libvpx/blob/6df3ec34557879fff673706f4a1d9fbd0f3a6f0e/vp8/encoder/onyx_if.c#L4946-L5010) +- [libvpx multi-resolution example](https://github.com/webmproject/libvpx/blob/6df3ec34557879fff673706f4a1d9fbd0f3a6f0e/examples/vp8_multi_resolution_encoder.c) +- [Windows process cycle measurement](https://learn.microsoft.com/en-us/windows/win32/api/realtimeapiset/nf-realtimeapiset-queryprocesscycletime) diff --git a/docs/research/node-local-media-design.md b/docs/research/node-local-media-design.md new file mode 100644 index 0000000..d3a9885 --- /dev/null +++ b/docs/research/node-local-media-design.md @@ -0,0 +1,474 @@ +# Node-Local Media Design Proposal + +Date: 2026-09-07. [ADR-0013](../adr/0013-embedded-node-local-media.md) accepts the +model and highest-demand policy. This file proposes implementation details, not +a claim that the replacement has shipped. The work includes embedded STUN/SFU +and a shared Native/SFU media module, not just Native encoding improvements. + +The 2026-09-08 [WebRTC encoder-pool review](./webrtc-encoder-pool.md) qualifies +this candidate: fixed output construction and forwarding allocation do not +include Browser encoder resource adaptation. The maximum-demand ownership model +remains accepted; the implementation below is not an accepted quality-parity +boundary. Review the proposed reuse point before adding compensating policies. + +## Recommendation From The Experiments + +Use one bounded output group with the maximum-demand envelope as the first +integration candidate. Existing framework capacity/layer selection chooses what +each child can receive; the group keeps outputs through the highest needed +level active. Avoid an exact-demand warm pool, per-child encoder, arbitrary +per-bandwidth representation or new optimization scheduler. + +This is a pragmatic initial policy, not proof that exact demand is inferior: + +- The [Native VP8 probe](./node-local-encoding-probe.md) measured about 26% more + CPU cycles for envelope than exact-cold on the same demand trace. Extra low + encodes are real work, not free efficiency from the word simulcast. +- The [VP8/H264 decode probe](./node-local-browser-probe.md) found very small + finite-set management cost and some backend-sensitive H264 activation delays. + Envelope keeps useful fallbacks ready but cannot eliminate all cold starts. +- Naive codec retention has timestamp/rate-control consequences. Mature codec + rate-control/activation ownership is worth more than avoiding a few local + initialization calls with another home-grown state machine. + +Exact demand remains a comparison policy in the same experiment, not a second +product mode. If hardware measurements show the unused-layer cost materially +hurts game cadence, change the group activation rule rather than building a +parallel transport. Neither policy may degrade the healthy input to satisfy one +weak child. Real-time playback takes priority over encode-count reduction. + +## Boundaries And Flow + +```text +capture or received encoded source (existing generation owner) + -> suitable encoded representation -> child transport + -> missing lower output + -> one shared decode if required + -> platform scale/encode group + -> shared encoded representations -> child transport + +each child's framework estimator/selector -> direct-child demand +direct-child demand -> group activation +input-limited path -> existing bounded topology operation +``` + +- **Source owner:** capture settings, media clock, selected codec and generation. + Turning off an unused output does not alter capture or discard best input. +- **Output group:** bounded representation slots from the selected media + component and Host ceiling. One owner writes codec parameters; children do not + independently overwrite the shared encoder. Reuse immutable encoded access + units and reference-counted surfaces. Do not create a generic cache/service. +- **Child transport:** its own congestion estimator, packet identity, recovery + feedback and chosen output. A source slot is not a PeerConnection. Changing + layers must not require a new route or a new PC for ordinary adaptation. +- **Router:** connectivity and input-quality opportunity. It never adjusts + codec settings. Healthy transport of already-poor input is not automatically + a good route; better input/reparenting still needs real candidate proof. + +An all-healthy relay retains zero new encoding: its received H is already +available. It does not create an encoder group just to keep unused fallbacks. +Only a missing demanded representation starts local derivation. Within that +group, compare/activate the agreed envelope without duplicating received outputs. +Leaf nodes create no outbound codec group. + +A Host using envelope may encode fallback layers even when both direct children +currently take H. This trades the literal one-Host-encode ideal for readiness; +it must not be described as both policies having identical healthy-case cost. +It also must not force those fallback bytes through every child link. Standard +SFU simulcast ingest can carry several layers in one publication; its aggregate +uplink budget must be respected. Encoding standby output locally and uploading +all layers are distinct costs. + +## Minimal Lifecycle + +1. Source/share generation owns available encoded slots and at most one local + derivation group. Existing lifecycle closure retires all owned resources. +2. Child join, departure or a framework-selected target change updates demand. + No room-wide survey, periodic rebalance or subtree demand recursion. +3. A newly needed slot is activated through the codec's normal controls. Keep + the previous usable output until a decoder-safe target frame exists, subject + to the transport's real congestion budget; never enqueue unlimited old video. +4. Target switching uses mature keyframe, RTP sequence/timestamp and codec + descriptor handling. Same output chunks can feed multiple child transports; + each transport still owns its packet headers and recovery. +5. Reduced demand stops unused upper outputs through the framework's normal + deactivation semantics. Any debounce belongs to that owner, not another + Screener timer beside it. No application-selected timer values in this draft. +6. Source replacement/failure/stop invalidates the same generation. No dormant + warm pool survives and no old callback can resurrect an encoder. + +Shared decode/scale must not block a ready encoded sibling. Each output's work +and each child's transport use bounded queues; control never waits for all +children to consume a frame. Queue/drop behavior must preserve codec dependencies +through the media library, not drop arbitrary packets or overwrite media clocks. + +### Complete-Frame Switch Boundary + +The first Native boundary uses complete encoded access units and one persistent +Pion packetizer per selected output stream. Pion owns fragmentation, packet +sequence and VP8 PictureID; codec recovery includes H264 SPS/PPS and IDR, not +an arbitrary packet labelled as a keyframe. + +The source owner arms a child-local cutover at an actual input frame, before +any representation of that frame is emitted. Older incumbent frames remain +eligible. At the fence, that child waits for the target recovery frame; siblings +continue independently. Target encoder preparation precedes the fence, and its +normal keyframe/recovery mechanism owns failed output. No guessed future PTS, +global all-output barrier, extra timeout or timestamp rebasing is introduced. +Canceling after incumbent reference frames were skipped also requires recovery. + +This boundary does not handle arbitrary already-arriving RTP streams. In that +case a target keyframe can arrive after the incumbent with the same PTS; merely +dropping `PTS <= last` can starve upgrades. A mature RTP projection/munger or +an explicitly controlled input-frame boundary is required. In particular, +`BeginGeneration` is for capture/source replacement, never layer switching. + +## Reuse Map + +| Existing module / primary source | Reuse or learn | Integration limit | +| --- | --- | --- | +| Screener `mediaedge.Source`, `Edge`, Native capture generations | Encoded-source ownership, Pion connections, existing share/route identity | Static RTP broadcast alone does not solve switching sequence spaces or codec descriptors | +| [Pion GCC](https://github.com/pion/interceptor/blob/main/pkg/gcc/send_side_bwe.go) and RTP/RTCP interceptors | Existing mature feedback and bandwidth estimate | Current Screener path only observes GCC and uses a no-queue pacer; selection/pacing is not already implemented | +| [LiveKit Dynacast manager](https://github.com/livekit/livekit/blob/v1.13.6/pkg/rtc/dynacast/dynacastmanagervideo.go) | Highest-needed activation and asymmetric demand handling | Apache-2.0; reference, not import the entire service or its downstream-node aggregation | +| [LiveKit Simulcast selector](https://github.com/livekit/livekit/blob/v1.13.6/pkg/sfu/videolayerselector/simulcast.go), [RTPMunger](https://github.com/livekit/livekit/blob/v1.13.6/pkg/sfu/rtpmunger.go), [prober](https://github.com/livekit/livekit/blob/v1.13.6/pkg/sfu/ccutils/prober.go) | Keyframe switching, packet continuity, bounded recovery probes | Valuable source, but imports protocol/buffer/logger infrastructure; not a demonstrated tiny drop-in | +| [libwebrtc VP8 adapter](https://github.com/webrtc-mirror/webrtc/blob/main/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc) | Coordinated libvpx encoding, SetRates, active-stream keyframes, prepared scale buffers, separate codec and media timestamps | BSD-style; references many WebRTC types. Reuse the underlying libvpx API before copying a large wrapper | +| [libwebrtc H264 adapter](https://github.com/webrtc-mirror/webrtc/blob/main/modules/video_coding/codecs/h264/h264_encoder_impl.cc) | Simulcast OpenH264 contexts, bitrate updates, scale-buffer reuse and keyframe/SPS discipline | Software reference, not proof that hardware multi-session cost is identical | +| Existing MFT/VideoToolbox/GStreamer/libvpx boundaries; [libyuv](https://chromium.googlesource.com/libyuv/libyuv/+/refs/heads/main/README.md) | Keep working capture adapters; native conversion/scaling and codec control | Received-source decoding and GPU surface sharing remain real work; don't replace working capture wholesale | +| [GStreamer tee](https://gstreamer.freedesktop.org/documentation/coreelements/tee.html) / [queue](https://gstreamer.freedesktop.org/documentation/coreelements/queue.html) | Shared frame ownership with branch isolation | Architecture reference; default queue sizes are not accepted latency budgets | + +inLive remains an embedded forwarding candidate, not the answer to derivation +or a proved adaptation replacement. Its inspected layer sender is coupled to +Client, track, bitrate-controller and packet-map owners; don't copy it as an +independent universal selector. No new dependency or upstream implementation +source was copied in this design pass. Preserve upstream licenses/notices when +an actual module is chosen; Screener's MIT does not replace them. + +Parallel inLive API inspection found work to validate before selecting it: +publisher-demand change notification is not public; `AddRelayTrack` installs a +no-op PLI callback; the layer sender has its own packet mapping rather than a +standalone codec-neutral switch API. Its inspected timestamp expression subtracts +the layer origin twice. Manager room removal/locking, failed-PC closing and +logger injection also need focused reproductions. These are candidate-library +risks, not claims about deployed Screener bugs or reasons to expand the Native +module into an SFU fork. + +## Required Before Production + +- Select a bounded codec/representation and packet-switching adapter backed by + actual RTP decode, loss/recovery and congestion tests, not this local probe. + Native H264 hardware and 60 fps resource ceilings remain unmeasured here. +- Prove behavior below the lowest nominal layer's bitrate and under encoder + overload. A fixed simulcast set alone is not complete adaptation. The selected + media component must still own usable bitrate/frame-rate reduction, recovery + and packet pacing; do not declare parity merely because three sizes decode. + One child must not mutate the base or all shared slots to solve its deficit. +- Measure decode/scale/surface-copy cost on the real Native relay path and prove + a stalled derived output cannot block original high output. Browser preview + decoding is not automatically a reusable zero-copy Native frame source. +- Preserve capability-based mixed Browser/Native behavior; no Browser RTP + passthrough promise, Browser WebCodecs transport replacement, AV1/VP9 or codec + change during a share. +- Specify real low-input recovery through the single router, with fresh + candidate evidence. Do not add a score or assume current topology is optimal. +- Implement ADR-0013 without claiming ADR-0007's runtime has already changed. + The current runtime + remains intact until its replacement is accepted and verified; no hidden + experimental production branch or compatibility layer. + +This is an implementation proposal with explicit unverified boundaries, not a +completed feature or a promise that every codec/device benefits equally. + +## Implementation Sequence + +1. Establish the shared Native/SFU encoded-output boundary from current Native + packetization. Keep one per-child packetizer across representation changes; + prove source-timestamp, payload-descriptor and generation continuity before + adding codec control. Existing Native single-output delivery consumes the + common component immediately. +2. Implement shared representation activation and safe frame-boundary switching + against that boundary, preserving current source and connection fences. Audit + an SFU library's public switching APIs in parallel, not as a prerequisite to + the Native quality module or a reason to build a fork prematurely. +3. Connect Native decode/scale/encode groups and framework per-child estimation, + rate selection and pacing. Verify Host and relay independently, including + below-lowest-layer and constrained-encoder recovery. +4. Integrate STUN/SFU with the existing Go room/signaling/config owners and match + Browser publication/subscription. Remove external-service integration in the + coordinated acceptance boundary, not through permanent dual backends. + +This is one phase with staged verification, not four independent production +releases. Current STUN-only and single-output fanout probes are completed +components, not acceptance of the complete phase. + +## Current Implementation Boundary + +`internal/media/forwarding` owns the shared input buffers, typed single-video +allocation and outgoing transport adapter. Native Source/Edge use it. Pion owns +ICE, crypto and reports; unmodified LiveKit media components own send-side BWE, +allocation primitives, RTP projection and video retransmission. A bounded pacing +adapter owns packet admission/retirement; a single-video event driver adapts the +upstream probe lifecycle without another congestion formula or application timer. +The replaced local Group/ +Selection and receiving-PC sender-estimator registry are deleted, not retained +as alternate implementations. There is no mirrored counter summing unused layers. + +The parent reads child state at actual capture InputBegin. A controlled layer +switch uses the existing DownTrack resync before outputs for that input, retaining +its target; this prevents an old-layer packet winning the same presentation time. +Raw RTP has no such honest input boundary and uses ordinary mature projection, +without decode or whole-frame repacketization. Inbound video padding is consumed +by projection while outbound sequence continuity remains intact. Incoming sender +reports retain their source-clock relation; local codec outputs correlate to one +input anchor, not independently to each encoder's completion time. + +Local capture replacement keeps its RTP source and cumulative counters while +rebasing only the input packetizers. It must not also restart the RTP projector. +Unconnected edges reserve capacity without asking for output. Existing audio +settings or observed relay-audio bytes reserve that portion of the PC budget. + +Capture producers now implement the coordinated v5 multi-output boundary below; +the packaged runtime has not been physically accepted. A bounded real libvpx +fixture supplies three outputs to the +same Native source API: 96 encoded AUs, two unchanged Pion connections, 40 selected +frames each, and 40/40 Chrome WebCodecs decodes at the expected dimensions and +timestamps. [Recorded result](./data/node-local-group.json) excludes payloads. +This is not a live Browser WebRTC jitter-buffer, hardware or network-loss gate. + +Whole-source/engine teardown closes every snapshotted PC before awaiting shared +write retirement; a deterministic held-write-lock test covers that ordering. +Windows/Linux H264 producers mark independent recovery only with SPS/PPS/IDR; +macOS prepends its format's parameter sets and verifies an IDR. The Windows +sidecar compiles and links; Linux compiles in the retained 1 GB/2 vCPU VM. +Updated macOS code still needs an SDK build. No desktop capture was started. + +The two-output packetization ablation measured 9.30-10.56 microseconds per 20 KiB +AU versus 4.90-5.29 for one packetizer on this machine, with about 53 KiB versus +26.5 KiB transient allocation. This is RTP work only, not extra encoding, +encryption or network cost. Keep independent output identities; no custom pool +or packet-header munger is justified by this measurement. + +### Native Producer Boundary + +The producer uses bounded per-output workers. Reuse one capture and +presentation decision; allow one processing and at most one pending raw input +per slot, not an unbounded encoded-frame queue. A slow MFT can currently wait +two seconds, so serially calling it for three slots is not an acceptable final +execution model. Derived-output validation is separate from user preset +validation, which intentionally rejects low representation dimensions/bitrates. + +Real input-frame begin and layer identity travel through the coordinated capture +envelope without a legacy reader. Encoding does not synchronously await a Go +acknowledgement. Control wakeup and retained quiet input allow output activation +and keyframe requests without reopening capture; a request remains pending until +the codec produces independent recovery. Normal VP8 empty output is not a fatal +capture error or an encoded frame; its previous drop policy remains unchanged. + +Each H264 slot needs its own activation object/MFT instance. Repeated +[`IMFActivate.ActivateObject`](https://learn.microsoft.com/en-us/windows/win32/api/mfobjects/nf-mfobjects-imfactivate-activateobject) +on the same activation returns the same instance, which would let independent +profiles overwrite one encoder. Share the D3D device/manager, not that instance. +Windows output workers now share one owned raw surface and isolate one processing +plus one pending input per encoder. A CPU-only check uses that production mailbox +with three real VP8 encoders; it verifies blocked-low isolation, bounded pending +work, sticky recovery, bitrate changes and activation retirement. A separate arm +of the same check verifies cancellation of a blocked production pipe writer. +macOS uses bounded VideoToolbox workers; Linux uses GStreamer raw tee/queues and +retains pipeline-level failure semantics. Compiler/CPU results are not driver +or desktop-media acceptance. + +Received media reserves the original upper slot and creates only its missing +lower output on actual direct-child demand. Pion depacketizers and the pinned +LiveKit SDK SampleBuilder reconstruct complete AUs, including a single quiet +frame; the ordinary Pion SampleBuilder waits for a subsequent timestamp and is +not used for this real-time input. H264 configuration uses Pion's Annex-B reader. +The derived clock comes from the original sender report; decoder completion does +not establish a new timeline. Compression loss requests normal throttled PLI. +An overflowing bounded decoder input retires that output rather than blocking +raw forwarding or silently dropping reference frames. + +The explicit `TestRelayDerivationFixture` uses the existing real 640x360 VP8 +fixture and the actual Windows encoded-input sidecar. It proves zero decoder +startup for healthy raw forwarding, a shared 320x180 output for lower demand, +live preference replacement without closing the healthy edge, and process +retirement. This short synthetic codec run does not prove desktop capture, +hardware H264, Browser decode, game quality or public-network recovery. +Cold outputs use the framework's existing four-second screen-share bitrate +measurement window; the acceptance trace includes it rather than substituting +configured bitrate for a measurement. + +Lowest-representation codec budgets now consume actual child video budgets; +they do not change the original or higher sibling outputs. Hardware overload, +extremely low bandwidth, real-game recovery and physical overhead still need +acceptance. A forwarding SFU does not invent a server-side transcoder. + +The pinned full publication allocator has no public per-RID pause setting; +globally disabling pause would also force unaffordable upper encodings and is +not used. Native publication retains the framework's pause behavior when no +usable output fits, and below-lowest-layer parity remains unverified. + +The real VP8 relay experiment at 6 Mbps -> 120 kbps -> 6 Mbps exposed a pause +while a live lowest encoder was being reduced below its nominal rate. Source-RTP +framing is now subtracted using actual access-unit/packet byte ratios and the +existing tracker window. The A/B changed only LiveKit's `allowPause` input +for an available lowest representation owned by a live local codec; raw forwarding +and zero-budget output retain pausing. Codec ownership is explicit and retired +with the encoder, not inferred from packets. The 30-second trace records actual +packet arrival and assembled-frame age separately; its receiver discards unresolved +older assembly on an independent VP8 keyframe. The original pause policy delivered +no low frames before release. Two unchanged local-codec-policy runs delivered +75 and 162 real 320x180 frames, with Chrome decoding all 573 and 620 weak-path +frames and all 891 healthy-path frames in each run. Each run used one derived +process. Low delivery began about 8-12 seconds after constraint; the second run +recovered high output 2.61 seconds after release. The healthy original remained +near 30 fps with about 26 ms packet age. Transition/retransmission age reached +seconds, so this verifies codec adaptation and eventual recovery, not smooth +low-bandwidth latency parity. The 500-packet assembly window also delayed frames +after an unrecoverable sequence hole: restored primary packets arrived in about +26 ms while assembly lagged over 20 seconds. Production relay input and the test +receiver now share recovery-boundary assembly: a newer codec-verified recovery +timestamp retires unresolved older data, retaining complete H264 configuration. +Focused RTP checks cover VP8/H264 loss, duplicate recovery packets and timestamp +wrap without another timer or packet queue. + +### Native Feedback Probe + +The explicit [GCC/TWCC probe](../../internal/client/mediaedge/testdata/gcc_feedback_probe.go) +and [recorded traces](./data/node-local-feedback.json) +use the pinned Pion interceptor v0.1.47, with its thresholds unchanged. Two +40-second memory-transport traces use a 6 Mbps -> 500 kbps -> 6 Mbps link, +20 ms propagation, a 120 ms queue limit and 100 ms feedback. Source bytes follow +the estimator; one arm caps released output at 1.25 Mbps. It is not encoded +video, a Browser receiver, a real network or a prediction of game quality. + +| Cadence | Released low-capped output | Released budget-following output | +| --- | --- | --- | +| 30 fps frame bursts | Estimate 0.468 -> 4.334 Mbps; payload remains about 1.25 Mbps | Estimate 0.473 -> 4.397 Mbps; final receive about 4.195 Mbps | +| 5 ms packet budget | Final estimate 38.62 Mbps while payload remains about 1.25 Mbps | Final estimate about 1.87 Mbps | + +No-prober does not prove permanent low-layer lock. The frame-burst run recovers +over almost 30 seconds; app-limited estimates may also grow beyond actual link +capacity. Therefore merely forwarding GetTargetBitrate to a codec is not evidence +of complete layer recovery or pacing. Keep the framework controller intact and +compare the selected complete media adapter against the same release trace. + +Run the probe explicitly from the repository root, optionally with +`-frame-bursts`; it writes only ignored `build/embedded-media` diagnostics. It is +not a default unit/CI task. The observed cadence sensitivity is a reason to test +the real packet path, not to introduce a Screener congestion score. + +### Embedded Service Composition + +Hosted configuration supplies one concrete IPv4 STUN listener on UDP 3478, +or three on 3478/3479/3480 when prediction is enabled. `STUN_LISTEN_HOST` +defaults to `0.0.0.0` and is separate from HTTP binding. Existing `STUN_URLS` +remain advertised discovery addresses: their DNS may intentionally differ +from the Web origin, so they must not be resolved as local bind addresses. +The Local Client constructor supplies no listeners; using public discovery +does not turn a Client into a public STUN service. + +The application binds HTTP and every required UDP listener before opening the +room database or accepting signaling. Failed startup returns all sockets; +the existing Close/End owner retires STUN too. No service supervisor or second +availability flag is needed. Deployment removes coturn in the final coordinated +cutover, not while the media implementation is still under acceptance. + +The implementation candidate uses a thin Screener SFU with LiveKit **media-core +components**, without its RTC room or service. A stock-Pion compile and actual +ReceiverBase/DownTrack projection pass; two receiving PCs also demonstrate an +unaffected high-output sibling and high/low/high allocation under forced budgets. +That is not a measured-network congestion/retransmission pass. Compared with the +current mediaedge API executable, the stripped Windows API closure grows from +8,920,576 to 32,467,456 bytes. Narrow munger/selector imports still reach about +300 non-standard packages and omit the complete projection/allocator, so they do +not justify a separate partial extraction. inLive would require private-path +fixes and demand hooks beyond a thin adapter; do not maintain both candidates. + +Keep the dependency behind a shared media package. For one video per connection, +use the public provisional allocation primitives, with exact typed child state +read by its parent; do not parse DebugInfo or run a second multi-track allocator. +The candidate now uses LiveKit SendSideBWE and Prober through their public APIs. +Pacing reuses the upstream queue with admission capped by the existing packet +cache bound; canceled probe tickets remain charged until actually dequeued. +The pinned Pion pacer's 1.5 headroom applies only to pacing, not codec or BWE +budgets. Adapted probe lifecycle code retains Apache-2.0 attribution. Probe +promotion requires the upstream validity criterion and non-congestion signal; +an empty or inconclusive attempt cannot promote the estimator's initial ceiling. + +The explicit `TestTransportConstrainedDormantRecovery` virtual-network trace uses real +Pion RTP/TWCC through 6 Mbps -> 500 kbps -> 6 Mbps shaping. It measured a minimum +budget about 497 kbps, stopped the upper output, and restored sustained 1.20 Mbps +after release. Recovery occurred about 4.4 seconds after release in the recorded +run; this is a bounded local trace, not a success-rate or game-latency claim. +Ordinary focused checks cover NACK repair, no-RR probe admission, zero-write +termination and exact shutdown. Before runtime cutover, complete actual codec, +Browser and physical recovery checks rather than inferring them from this trace. +Desired maximum, available layers and current forwarding target remain distinct; +Dynacast is not proof that stopping every currently unselected high layer alone +provides recovery. + +For dormant output admission, the parent can prepare a layer when a child's +fresh framework video budget covers that layer's existing configured ceiling. +The active prefix includes this preparation demand, the library target and the +still-forwarded incumbent. This is our small activation rule, not a new +congestion estimator or a fabricated measured bitrate. A prepared output becomes +selectable only through real codec output and the normal library allocation. +Configured ceilings are conservative and do not replace bandwidth probing; +no cache of historical layer rates or additional polling owner is introduced. + +### Capture Contract For The Next Implementation + +Use the existing LiveKit 2.22.1 screen-share construction as the initial output +set: original plus half-size at the same FPS, quarter bitrate with its 150 kbps +floor. Align derived NV12 dimensions down to even values. The group supports up +to three outputs, but this step does not invent a third product preset. + +The private capture command/probe contract advances to v5. Video arguments retain +the source profile and append one `--output WIDTH HEIGHT FPS BITRATE` per output, +ordered low to high. All current producers/readers change together, without v4 +fallback. Output profiles are codec bounds, not user preset choices. + +SMED v2 has a 32-byte big-endian header: magic[0:4], version[4], kind[5], flags[6], +layer[7], timestamp100ns[8:16], duration100ns[16:24], width[24:26], height[26:28], +payload length[28:32]. Video carries its output dimensions. PCM/status dimensions +and layer are zero. Kind 5 is a zero-payload input-frame begin with dimensions/ +layer/flags zero and a valid source timestamp/duration; it precedes all outputs +for that input even when the active prefix is zero. Kind 6 reports an unavailable +output layer with a bounded UTF-8 diagnostic, zero timing/dimensions/flags. + +Both pipe directions use this envelope. Kind 7 carries one bounded ASCII control +command (at most 64 bytes), with all other header fields zero: `Q`, `K layer` +(`-1` means all outputs), `A activeCount`, `B layer bitrate`. Reuse the existing +command grammar instead of mixing a line reader with binary video on stdin. Keyframe +requests consumed at encode start are restored on failure; requests arriving +during encoding must not be erased when an older keyframe completes. Control +wakeup is I/O work, not another media-quality scheduler. + +Every slot owns its encoder/converter. Windows must retain one safe copy of the +WGC input surface shared by slot workers, not reuse a returned frame-pool texture. +Output writing is serialized at the frame envelope, not by waiting for all +encoders. Source timestamps/presentation are computed once per captured input. + +The same sidecar's encoded-input mode accepts complete H264/VP8 access units, +decodes once and feeds those same bounded raw-output workers. It is created only +for a missing demanded lower representation; healthy forwarding bypasses it. +Use system MFT/VideoToolbox/GStreamer decoding or the already bundled libvpx. +Compressed reference frames cannot use the raw mailbox's overwrite behavior: +loss or overflow requires normal decoder recovery and an upstream keyframe. +No additional local service, network port, selector UI or media clock is needed. + +### Reproduce The Group Gate + +Compile `native/capture/windows/encoded_group.fixture.cpp` with `vp8_encoder.cpp` +and the existing pinned libvpx build, using the same MSVC flags as the earlier +CPU probe. Generate its JSONL under `build/embedded-media`, then run: + +```powershell +$env:SCREENER_ENCODED_FIXTURE = 'build/embedded-media/encoded-group.fixture.jsonl' +$env:SCREENER_ENCODED_OUTPUT = 'build/embedded-media/encoded-group.received.json' +& .\build\embedded-media\mediaedge.test.exe '-test.run=TestEncodedGroupVP8Fixture' '-test.v' '-test.timeout=30s' +node scripts/encoded-group-decode.mjs +``` + +Open the printed loopback URL in isolated Chrome; the decoder gate saves its +result and stops its own server. Neither codec generation nor this browser gate +is a new default CI job. The synthetic fixtures remain generated build output. diff --git a/docs/research/webrtc-encoder-pool.md b/docs/research/webrtc-encoder-pool.md new file mode 100644 index 0000000..28166a1 --- /dev/null +++ b/docs/research/webrtc-encoder-pool.md @@ -0,0 +1,663 @@ +# WebRTC Adaptation And Encoder Reuse + +Reviewed: 2026-09-08. Source-backed review and synthetic VP8/H264 feasibility probes, +not an accepted engine replacement. [ADR-0013](../adr/0013-embedded-node-local-media.md) +still owns the direct-child/shared-output model; [TODO](../todo.md) owns work and +the dependent release hold. The question is whether retaining mature encoder +control while sharing compatible encoding work is smaller and more complete +than extending the current Native fixed-output candidate. + +## Pinned Baseline + +The installed Chrome is 152.0.7977.82. Its [DEPS][chromium-deps] pins libwebrtc +to `6f37672d358475cd17544121a12494da454d85fb`; WebRTC references below use that +revision, not a floating upstream branch. Source shows available mechanisms, +not which field trials or hardware encoder a particular runtime selected. +The Screener comparison uses the `3c192ddd` candidate, not production. + +## Actual Control Ownership + +```text +receiver transport feedback + -> GoogCC: network target, pacing, probes, congestion-window updates + -> Call / BitrateAllocator: per-media allocation + -> RtpVideoSender: transport, RTP, packetization and NACK/FEC costs + -> VideoStreamEncoder: layer rates and encoder feedback + -> VideoEncoder: encode, SetRates, encoded callback + -> RTP packetization / pacer -> network + +encoded QP, bytes, drops and completion timing + -> quality/bandwidth scaler and encode-usage resource + -> ResourceAdaptationProcessor / VideoStreamAdapter + -> source pixel/FPS restrictions -> adapted input / encoder reconfiguration +``` + +These are connected but different owners: + +| Owner | What it actually decides | +| --- | --- | +| [Transport controller][transport] / GoogCC | Estimates network capacity and drives pacing/probing, not a pixel-size ladder. | +| [RtpVideoSender][rtp-budget] | Computes the video payload budget after transport and protection costs. Network BWE is not the encoder's payload target. | +| [SimulcastRateAllocator][simulcast-rates] | Allocates among configured layers with activation minima/hysteresis and temporal rates. It does not derive their dimensions from BWE. | +| [VideoStreamEncoder][vse] | Applies target/adjusted rates, optional overshoot correction, queue/frame dropping and encoded feedback. One instance has one resource-adaptation chain, not one complete controller per RID. | +| [Resource manager][resources] | Uses trusted QP and drops, or eligible bandwidth-quality scaling, plus encode-time pressure. These paths depend on actual encoder capabilities and configuration. | +| [VideoStreamAdapter][video-adapter] | Requests fewer pixels or FPS according to degradation preference and resource constraints; a pre-encoded fixed tier is not required. | + +For example, the adapter can request at most 3/5 of the current pixel count or +reduce FPS to 2/3 under the corresponding preference. These are upstream rules, +not proposed Screener constants. The default [encoder minimum][encoder-api] is +320x180 pixels and can be overridden; no universal 108p floor is established. +VP8's [libvpx adapter][vp8] disables libvpx internal resizing because WebRTC owns +that work outside the codec. Turning on libvpx resizing alone does not restore +the omitted control chain. + +### Motion And Simulcast Are Not Synonyms + +Screener's `motion` hint becomes `kFluid` in the [Chromium sink][chromium-hint]. +[VideoRtpSender][sender-hint] then sets `is_screencast=false`; this follows the +realtime-video path even for display capture. Explicit degradation preference +still has precedence over the hint's default. Do not assume all display capture +uses WebRTC's special screen-content policy. + +At this pin, the [media engine][media-engine] permits its automatic quality +resizing path only when eligible and there is one configured SSRC or one active +stream. Multiple active simulcast outputs therefore do not each acquire an +independent version of that resizing path. The [simulcast encoder adapter][simulcast-encoder] +coordinates per-output codecs, rates, scaling and keyframes; it is not a pool +across independent PeerConnections. Adding a third preset cannot establish parity. + +## The Current Native Gap + +`nativecapture.ScreenShareOutputs` constructs original plus half-size outputs +at the same FPS. `mediaedge.Source.BeginFrame` chooses activation and lowers the +lowest encoder's bitrate from its consumers' budgets. Windows `OutputWorker` +keeps one fixed profile for its lifetime. At 1080p the lowest nominal output is +960x540, not a dynamically shrinking fallback. Bounded raw mailboxes isolate +overload but do not automatically reduce the requested encoding work. + +Pion supplies transport; LiveKit media components supply BWE, forwarding, +allocation, recovery and pacing. They do not supply the omitted libwebrtc raw +source/encoder resource-adaptation chain. The current source-RTP cost adjustment +also is not the full `RtpVideoSender` overhead/protection calculation. These are +concrete missing integrations, not evidence that either library is defective. +The existing constrained-codec recovery evidence remains valid within its scope, +but cannot establish Browser-equivalent behavior below the fixed lowest size. + +## Where A Pool Can Fit + +[VideoEncoderFactory][encoder-factory] is a real native injection point. Its +`Create(env, format)` returns a `VideoEncoder`; it is not exposed by Browser JS +and does not receive a source/stream identifier. A source-scoped integration +must establish that identity without guessing from dimensions or timestamps. +Pion currently has no libwebrtc encoder factory to override: retaining the full +chain requires a native-library integration decision, not only a new Go cache. + +Shared encoded payload buffers are supported by [EncodedImage][encoded-image]. +That makes payload reuse possible, not automatically correct. The narrow +adapter would have to preserve all of the following: + +1. **Compatible demands.** Codec/profile, source generation, dimensions, + cadence, rate allocation and reference state must agree. `SetRates` applies + until replaced; last-writer-wins, unconditional minimum, or unconditional + maximum would violate another consumer's quality or budget. Resolution-only + keys are insufficient. Exact compatibility may produce little reuse and + needs measurement before introducing any grouping heuristic. +2. **Compatible frame dependencies.** Each VSE can drop different inputs before + calling `Encode`. If A encoded frame F but B skipped it, a later shared + interframe may reference F that B never received. Skipping a whole input is + not equivalent to ordinary packet loss that NACK can repair. Splitting and + rejoining must use codec-safe recovery, not arbitrary callback suppression. +3. **Separate callback ownership.** Each virtual encoder owns initialization, + keyframe requests, release and its callback. Reuse immutable payload bytes, + not one mutable metadata object. Preserve each caller's RTP/input identity, + codec descriptors, QP and completion semantics. +4. **Honest resource feedback.** [Frame metadata][frame-metadata] matches the + caller's timestamp and measures its encode-start/completion interval. A cache + hit can appear cheap despite a busy shared physical encoder. Validate the + resulting resource decisions; do not fabricate CPU/QP evidence to keep reuse. +5. **Unadapted source availability.** [VideoBroadcaster][broadcaster] aggregates + minimum maximum-pixel/FPS restrictions across sinks; requested-resolution + aggregation is a separate path. A pool cannot recreate input detail already + removed upstream. Retain original capture and validate sibling isolation. + +Therefore the owner's abstraction is useful, but a transparent cross-sender +pool preserving every independent control decision is not established by the +factory API alone. The experiment below distinguishes that from a pool owning +the physical encoder's correction state. +Per-node encoding cost follows the number of compatible active representations, +not automatically a fixed three encodes regardless of downstream conditions. +Already-encoded Native relay/SFU forwarding must remain zero-encode when usable; +an encoder factory by itself does not implement that bypass. + +## Mature Precedent And Reuse + +Epic's [Pixel Streaming 2 migration guide][epic-migration] records removal of +its former one-stream/one-quality-controller sharing because of stalls and +freezes, recommending per-peer encoding or its SFU. The older +[settings guide][epic-settings] explicitly describes mismatched client quality +under that design. This is a warning against selecting one peer's conditions +for everyone, not proof that a compatible-output pool cannot work. + +Reuse upstream encoder factories, codec adapters, simulcast allocation, +resource adaptation and projection where their contracts fit. No reviewed +source supplies a production-ready pool with all five obligations above. +The WebRTC [test proxy factory][test-proxy] simply delegates to one encoder; +it supplies no callback/rate arbitration or frame deduplication. An embedded +forwarding SFU should keep the current mature forwarding components; it normally does not +encode, so sharing representations/demand is the common boundary, not forcing +every SFU subscriber through a codec factory. + +The libwebrtc [build guide][webrtc-build] describes a native GN/Ninja dependency +build. Its headers are not a stable Go plug-in ABI. Evaluate reproducible build, +update and package cost before replacing the Native media engine. WebRTC code +retains its BSD-style license and PATENTS grant, dependencies their own notices. +Epic's infrastructure MIT license does not license Unreal Engine/EpicRtc code. +This review copies no upstream implementation and adds no product dependency. +The opt-in probe links the pinned SDK only under the ignored build directory. + +## Executed Feasibility Probe + +The [opt-in source and build recipe](../../native/probes/encoder-pool/README.md) +use Shiguredo `m152.7977.0.2`, whose WebRTC revision exactly matches the pin +above. Its relevant vendor differences are disabled built-in H264 and disabled +pacer keyframe flushing; this VP8 encoder-only probe uses neither path. +Matching SDK headers/static library and official MSVC libraries/linker were +used. No desktop capture, ports, game, production service or CI job was involved. + +Two actual VSEs receive one synthetic 640x360@30 source through separate stock +VideoAdapters. Both receive identical 2 Mbps budgets/feedback cadence; B alone +gets 100 kbps for 15 seconds and then returns to 2 Mbps for 25 seconds. +Degradation preference is maintain-framerate to expose spatial adaptation. +This supplies encoder budgets, not a simulated full GoogCC/packet network. +The driver drains queues between inputs, so it is not a throughput benchmark. + +The factory starts with exact codec/rate compatibility and one cached access +unit per physical encoder. Different requests split, compatible streams rejoin +on a keyframe, and each VSE gets its own metadata/callback with shared payload. +Attempted input and successful reference identity are separate so a codec drop +is not submitted twice. RTT/loss are excluded only because the guarded realtime +VP8 L1T1 implementation's [DefaultTemporalLayers][vp8-default-temporal] ignores them. + +Three forms were exercised, plus an explicit disabled-adjuster ablation: + +| Form | Healthy 6-second physical Encode calls | Delivered frames per consumer | Result | +| --- | ---: | ---: | --- | +| Independent encoders, stock VSE correction | 360 | 180 / 180 | Control | +| Shared factory, separate unchanged VSE correction | 337 | 180 / 180 | Only 23 reuses; independent adjusted targets soon differ | +| Shared factory and one upstream corrector per physical encoder | 187 | 180 / 180 | 173 reuses, about 48% fewer calls in this phase | + +Both VSEs can have the same 2 Mbps logical target but ask their encoder for +different corrected rates, such as about 1.67 versus 1.59 Mbps. The upstream +[EncoderBitrateAdjuster][bitrate-adjuster] observes encoded size/timing to +correct overshoot; it is not a resolution selector or a receiver-supplied QP. +Disabling it only for diagnosis restored healthy reuse, locating the conflict. +The viable probe form instead instantiates that same library class per physical +encoder and feeds each physical output once. It does not delete correction or +choose one peer as quality authority. + +In the final paired full runs, both forms first reduced B at about 4.04 seconds, +reached 320x180, and first returned to 640x360 about 24.05 seconds after release. +A remained full-size at about 30 fps. The pooled form resumed shared output after +recovery and reused 87 frames in the following 3-second, single-skipped-input +phase. Both groups fully retired, with at most two live physical encoders. +Repeated earlier runs showed the same broad transitions. This does not promise +fast or seamless recovery: the measured upward delay was substantial in both. + +Weak-phase delivery was not identical: the final control decoded 154 frames +(10.27 fps, 85.3 kbps), versus 142 (9.47 fps, 95.1 kbps) for the pooled form. +Splitting/rejoining also produces extra keyframes. Do not infer perceptual or +congested-network parity from matching dimensions and transition times. + +WebRTC decoding checked usability; raw libvpx with postprocessing disabled +checked matching-payload reference correctness. The [stock VP8 decoder][vp8-decoder] +enables history-dependent MFQE on x64, so comparing its postprocessed pixels +after different histories produced one misleading mismatch during development. +The corrected final full pooled run compared 288 shared payloads with zero raw +pixel mismatches. Removing reference-continuity protection in a negative +control produced 29 mismatches despite successful decoder calls. The guard is +necessary, not decorative defensive code. [Structured results](./data/webrtc-encoder-pool.json) +contain no frame payloads or private identifiers. + +### Verdict And Remaining Scope + +- **An unchanged factory cache alone does not meet the goal.** Independent + encoder-side correction fragments otherwise equal nominal demands. +- **The shared-encoder abstraction passes a narrow executable feasibility + check when physical correction state shares that owner.** It reuses real + encoding, retains stock spatial adaptation and can split/rejoin safely in + this VP8 case. No engine replacement has been selected or integrated. +- The group invokes correction on actual frames, with rate-write deduplication; + stock VSE does so on rate/configuration/FPS updates. The algorithm is reused, + but this cadence change must not be described as an otherwise identical move. +- Default VP8 trusts its rate controller, so pre-encoder media-optimization + dropping is inactive here. Other encoders need the corresponding drop feedback: + it is outside VideoEncoder's callbacks. Codec drops must not impersonate it. +- Per-VSE encode timing still differs for a physical encode and a cache hit. + H264 hardware/asynchronous execution, actual resource overload, transport + overhead, received-source bypass and perceptual parity remain unverified. + These are integration gates, not reasons to disable mature adaptation. + +The reusable boundary is an encoding instance plus its encoder-side state, not +just `Encode(frame)`. Before product replacement, resolve those remaining +feedback owners through upstream interfaces; do not accumulate quality formulas +or silently reintroduce per-child encoders as the ordinary product model. + +## Cross-Level Reuse + +The shared boundary already exists in `mediaedge.Source`. A Native Receiver +feeds received packets directly through `Source.WriteRTP` into LiveKit buffers +and DownTrack projection. It does not call a decoder or encoder to forward an +available representation. Local capture/derivation instead supplies complete +codec output through `Source.WriteVideo`. The pool fits behind that second +entry, not in front of every received packet: + +```text +received original -> existing per-child RTP forwarding -> healthy descendants + -> missing direct-child output only + -> one real decode -> VSE / shared physical encoder + -> Source.WriteVideo -> child transport +``` + +The factory's first 180 real VP8 output AUs were exported and replayed through +five actual loopback Pion connections: Host to R1 to R2, with one leaf attached +at each source. Both Native receivers had normal derivation capability and +profiles enabled. The final trace delivered 180, 167 and 167 contiguous AUs at +the three leaves, starting at indexes 0, 13 and 13 respectively. Every received +AU matched an original SHA-256 hash, beginning with independent recovery; no +derivation run was observed at either relay. Earlier runs began at the later +recovery index 19. The prerecorded source cannot answer live PLI requests, so +this startup interval is not a capture/join-latency measurement. + +An extension to the existing Native derivation fixture separately verified +640x360 original forwarding alongside one 320x180 derived output, then two +compatible low consumers sharing the same live derivation process. Retiring +one low consumer left the other receiving new frames; retiring the last stopped +the process. The original input stayed encoded. Target layers/budgets in this +fixture are forced to test ownership, not prove automatic network selection. +Both checks passed together and cleaned up. The test awaits Pion's graceful +completion outside callbacks rather than equating concurrent `Close()` return +with completed transport teardown. [Recorded evidence](./data/webrtc-encoder-chain.json) +contains counters and hashes, not media or private connection identifiers. + +These measurements establish zero additional encoding on the observed healthy +Native hops, and local sharing of a genuinely missing output. They do not yet +connect the new VSE pool to that native derivation process: the source-pool, +network replay and existing-codec derivation are separate verified boundaries. +The original AUs were already decoded in the C++ probe; byte-identical contiguous +forwarding is not another Browser playback, original-clock or loss-recovery gate. + +"One encoding for the whole tree" requires one active source representation +and Native/SFU intermediate forwarding nodes. Pure Browser intermediate nodes +still encode through their normal senders; Browser leaves add decoding only. +If Host keeps pre-encoded lower standby outputs, those encodes must also be +counted. The current packaged producer still constructs original plus half-size; +neither this one-output replay nor a no-extra-hop result silently changes the +accepted envelope policy. The pool probe also has occasional split/rejoin work, +so its healthy case is close to one encode per frame, not an unconditional bound. + +### Minimal Integration Direction + +Keep the existing Pion/LiveKit transport, estimator, pacer and projection per +edge. Codec-only `VideoStreamEncoder::OnBitrateUpdated` accepts an external +budget without constructing a GoogCC controller, Call or PeerConnection. +`relayPlan` already recognizes missing output from observed positive bandwidth +and paused/lower forwarding demand. It does not need to choose pixel dimensions; +VSE and stock `VideoAdapter` can make that decision from the actual raw input. +A healthy original consumer needs neither VSE nor a fabricated raw frame. + +The owning source can reuse `OnDemandChanged` to signal work, never reenter +transport synchronously under its locks. This callback is not currently wired +by Native `NewEdge`; current relay demand is reevaluated at input markers. +Budget-to-payload conversion, dynamic output association in bounded source slots, +physical resource feedback and decoder-safe switching remain concrete integration +work. Connecting those owners is narrower than replacing the transport engine, +but its total change/maintenance cost is not yet measured. A low-quality received +source still cannot manufacture missing detail; higher input or the existing +bounded topology operation must supply it. No subtree quality minimum or new +congestion algorithm is introduced by this direction. + +## Scheduling And Resource Follow-Up + +The initial probe drained each VSE queue between consumers. A later +`--realtime` arm keeps the 30 fps source cadence without that drain and flushes +queues only at phase boundaries. Its healthy six seconds used exactly 180 +physical encodes for two 180-frame receivers, with zero decoded-hash mismatch. +This improves the fixture's scheduling realism, not a one-encode guarantee for +all devices, rates or source histories. + +The bounded `--latency-probe` inserts 45 ms before each physical VP8 encode at +640x360, proportional to pixel area at lower sizes. It is a synthetic service +delay, not CPU saturation, desktop capture or a hardware-driver test. The +prototype's registry mutex serializes physical groups, so this arm stresses +ownership and is not a performance comparison with independent parallel codecs. + +Under that delay, the two-VSE shared-codec form developed repeated independent +input-drop divergence. During 30 seconds, A/B decoded 391/378 frames and produced +146/149 keyframes; the full trace created 115 codec groups and made 218 group +changes. Reference protection kept the decoded payloads correct, but this churn +is not acceptable product behavior. Both returned to about 30 fps after release. +This is a counterexample to accepting the factory-cache boundary from its +healthy result alone, not proof that all shared encoders must behave this way. + +A `--shared-pipeline` control instead gives both consumers one complete stock +VSE/source-adaptation/encoder pipeline. It has no per-consumer encoder factory +cache and leaves upstream overshoot/resource handling enabled. In the same +delay trace, both decoded the same 484 frames with no new keyframe during the +delayed phase; the whole trace created three encoder instances, only for normal +configuration changes, and no split/rejoin cycle. Both consumers' resource +fields in this control refer to the same pipeline, not two independent sensors. + +Neither delayed arm delivered a smaller picture until after latency was +released; both subsequently showed 480x270 and recovered full size. Do not claim +complete overload adaptation or perceptual parity from these runs. The useful +result is narrower: shared input admission and shared resource ownership avoid +one source of duplicate-controller churn. Different-child-budget grouping, +transport coupling and actual hardware still need evidence. +[Structured traces](./data/webrtc-encoder-latency.json) retain those limits. + +The pinned library also has `EncodeUsageResource`, `OveruseFrameDetector` and +`BroadcastResourceListener` for one resource feeding multiple send streams. +VSE exposes `AddAdaptationResource` but no public removal counterpart; registering +a new resource after each group change would retain old restrictions until +Stop. A stable per-VSE forwarding resource can avoid that, but is additional +coordination, not a reason to add it before the simpler shared-pipeline boundary +has been evaluated. Do not disable local CPU detection unless an honest shared +measurement replaces it, or replay a cached frame's timing as physical work. + +Windows H264 need not be rewritten: existing `LiveEncoder` already has bounded +Encode, SetBitrate and retirement around MFT. A shared extraction plus an adapter +can preserve that implementation. Truthful codec information, requested FPS, +actual QP availability, and dynamic output dimensions remain integration work; +the old fixed `OutputWorker` policy is not a requirement to restart WGC capture. +The following H264 step first verified that extraction and opt-in adapter; +the subsequent product attachment is recorded below. + +### Hardware H264 Adapter Check + +The existing Windows `LiveEncoder` and its MFT/device helpers now have one +shared implementation in `native/capture/windows/h264_encoder.{h,cpp}`. Both +the capture binary and the opt-in probe link it; WGC, source selection, audio, +profile application and capture retirement retain their existing owners. +The probe adapter converts synthetic I420 to NV12 and uploads it for MFT. +This validates the native encoder interface, not a zero-copy capture path. + +One complete stock VSE owns rates, pre-encoder dropping, correction and resource +feedback. The adapter reports untrusted hardware rate control and unavailable +QP, rather than claiming the VP8-specific cache feedback applies to H264. +It preserves callback timestamps and shares immutable encoded bytes. SetRates +uses the configured/requested-FPS bitrate compensation in Chromium's +[Media Foundation adapter][chromium-mf]; zero budget pauses instead of writing +an invalid MFT bitrate. WebRTC's [codec initializer][codec-initializer] owns +default H264 settings and the single temporal layer. + +The bounded 640x360@30 run produced 174 physical encodes during six seconds, +delivering 174 byte-identical AUs to each consumer. First output took about +275 ms. Removing B left A receiving 60 further frames over two seconds. +One physical encoder was created and all resources retired; there was no +split/rejoin. The pinned SDK has no H264 decoder, so its C++ decoded counters +are explicitly zero. Two Chrome WebCodecs decoders subsequently each decoded +the exported first 180 AUs with the expected dimensions/timestamps and no +reported errors. The last six exported AUs followed B's retirement; this is +bitstream validation, not a claim that both actual consumers received 180. + +[Structured results](./data/webrtc-h264-pipeline.json) retain that narrow scope. +These initial checks do not establish independent weak-child adaptation, +overload, actual capture throughput, transport overhead or Client integration. + +The concrete product attachment is `OutputWorker::Run`, shared by WGC and +encoded-input derivation. Its fixed-size/FPS/direct-rate block can become a +complete VSE while retaining the bounded mailbox, generation fence and +stop/join lifecycle. Bind the H264 adapter to the existing D3D device and reuse +`FrameConverter`/owned decoded NV12, rather than introducing the probe's CPU +round-trip into normal capture. Preserve one timestamp domain across Begin and +AUs: rounding a 100 ns capture timestamp down to WebRTC microseconds can put +the first AU before its Begin anchor. Dynamic output sizes must update the +forwarding format metadata as well as `Source.SetFormat`; ordinary adaptation +must not rebase the whole source generation. + +The remaining grouping boundary is not transport-independent bookkeeping yet: +the initial `OutputPlan`/`relayPlan` collapsed demand into one lowest bitrate. +The grouped integration below replaces that owner. The SDK's bundled libvpx +and notices now replace the separate capture dependency. Those were concrete +implementation costs, not solved by the healthy two-callback check above. + +### Windows Product Attachment + +`OutputWorker` now uses one complete stock VSE/VideoAdapter per shared local +output, for both WGC and encoded-input derivation. The original capture, bounded +mailbox, generation checks and cancellable pipe writer remain. H264 receives +same-device NV12 directly; stock VP8 uses I420 readback. The returned AU retains +its exact input timestamp rather than rounding it through WebRTC microseconds. +The source publishes adapted dimensions through an immutable metadata snapshot; +calling LiveKit's full track-info update for each size change would reset every +active layer's bitrate tracker and was rejected by a focused regression. + +The product build now links the same pinned SDK, including its libvpx, instead +of a second standalone libvpx build. The Windows capture executable is about +8.0 MB and imports only Windows system DLLs/API sets. Cached SDK and official +MSVC archives total about 816 MB, not package payload. SDK notices are retained +in the package; no new daemon or runtime service is added. + +The opt-in `TestAdaptiveOutputFixture` sends synthetic encoded input through the +actual capture executable: six seconds at 2 Mbps, fifteen at 100 kbps, then +restoration to 2 Mbps. VP8 delivered 178 full-size healthy frames, 94 reduced +frames during the limited interval and 26 full-size restored frames within +the first 25 seconds after release. Source timestamps remained ordered. + +H264 exposed two interface differences. Unconditionally marking QP untrusted +selected the bandwidth-quality scaler rather than Chromium's usual parsed-QP +path. The adapter now allows VSE's existing H264 parser and uses upstream +24/37 quality-scaler thresholds, without a custom controller. The existing MFT +implementation also forced a one-frame VBV at initialization. Updating only +mean bitrate left about 118 kbps output after restoration; attempting to change +VBV dynamically failed driver readback and was removed. Chromium does not set +that VBV override. Deleting the override and its exact-value assertion restored +about 1.9 Mbps while retaining CBR and low-latency mode. + +The remaining delay was visible in upstream logs: parsed QP fell to 22-25 but +the low-QP smoother remained above the 24 upscale threshold. In a 76-second run, +resolution returned via 480x270 to 640x360 around 50-55 seconds from start, +about 29-34 seconds after budget restoration. Counts were 178 healthy full-size, +265 limited reduced-size and 686 restored full-size frames, 2,105 total. No +connection/pipeline reset or new timer forced recovery. Temporary internal +tracing was removed after this diagnosis. This is eventual recovery, not a +low-latency recovery guarantee; Intel-specific QP capability quirks and actual +overload still need hardware evidence. + +Auto VP8 measurement now uses the actual VSE path rather than benchmarking an +encoder with different live settings. H264's existing successful-hardware probe +path remains. The four-second decision budget is unchanged. Current grouping +at that point still folded multiple weak consumers into one low budget; the +following integration addresses that independent requirement. + +### Compatible Output Groups + +The source now retains per-consumer demand instead of taking a minimum across +weak children. Effective connection budgets are bounded by the same output +ceiling; equal requests share one complete pipeline, while differing requests +are isolated. The group's rates change in place, so a changing estimate is not +a new encoder identity. Stable slot selection avoids changing groups due to +map iteration. Two logical representations remain visible to a connection: +original plus its assigned adaptive output. Additional physical codec slots +are bounded by source admission and do not become new product quality presets. + +The existing forwarding source is reused per group. This duplicates bounded +RTP buffering/projection of the original, not original encoding or decoding. +LiveKit's `DownTrack.SetReceiver` retains the transport; the adapter preserves +its own callback wrapper and clears old input clock references before retarget. +Detached packet, sender-report and closure callbacks cannot mutate the new +attachment. A source handoff follows an accepted recovery AU and requests +recovery for the new attachment; pending members keep their current output. +Unneeded physical codec slots stop, and a Native relay still owns one decoder +process for all locally derived groups. Source-generation changes and rejected +timestamps are fenced before attachment changes. + +The internal capture v7 contract supports encoded AUs up to 4 MiB and independent +per-slot activation in place +of the old prefix count, permitting equal-ceiling outputs with different +budgets. All three platform producers implement that contract. Windows builds +and normal tests pass; Linux production compilation and CPU-only GStreamer +slot retirement checks pass. The macOS change is source-reviewed, not SDK or +physical acceptance. No Browser/server signaling change is needed for private +encoder grouping; matched Client and capture artifacts are required. + +Actual VP8 relay checks exercise two equal 300 kbps consumers sharing one +derived output, splitting to 80/300 kbps and rejoining at 300/300 kbps. Both +consumers keep receiving and the decoder process remains the same. Independent +consumer and final-process retirement pass. A separate 1280x720 moving-input +trace keeps one child at a 1.25 Mbps derived budget while the other receives +80 kbps: the latter reaches 320x180 and the former stays at 640x360. Over the +roughly 20-second weak phase they receive 45 and 596 frames respectively; +Chrome WebCodecs decodes every exported frame with its expected dimensions +and timestamp. This proves spatial isolation and valid delivery, not weak-path +cadence parity. Budgets in these grouping fixtures are explicit transport +inputs; the separate shaped-network gate owns automatic estimator evidence. + +The full H264 Windows Host/Viewer gate also passes live and paused profile +changes, background-profile recovery, source replacement and restart while +preserving media identity. SFU retarget has real loopback tests for clock, +per-RID counters, recovery and late old-source callbacks. Complete network, +hardware-overload, multi-room and platform acceptance remain distinct work. +The Native H264 embedded-SFU gate additionally decodes 300 720p frames, applies +a live 480p change, retains decoded Opus and closes publication/capture/UDP +resources. Its Vite server disables file watching: scanning the growing ignored +build tree had blocked the test's in-process event loop and CDP navigation +before media started. No Browser media deadline was relaxed to obtain the pass. + +The subsequent automatic-BWE relay checks are mixed: one 120 kbps constrained +run retained a roughly 49 kbps estimate throughout the twelve seconds after +release; an instrumented repeat, with no runtime correction, returned to the +original 640x360 output about 1.3 seconds after release. That successful trace +shows valid higher-output demand and the stock congestion guard clearing, not +proof of why the earlier run stayed low. The fixture's 6 Mbps value is link +capacity, not payload rate: its replayed original is roughly 157 kbps and its +bounded queue makes keyframe/retransmission bursts relevant. Temporary probe +instrumentation was removed. Observation files now follow each received-output +path and counters follow the actually attached group, preserving failed runs +and avoiding stale-group attribution. This is an unresolved repeatability +boundary, not evidence for a new recovery timer or a permanent deadlock. + +A 75-second follow-up on the 4,096-packet candidate kept the same 5-to-18-second +120 kbps constraint and restored the original at 19.535 seconds, 1.535 seconds +after release. The stock congestion guard cleared, a 200 ms padding probe +requested 315,957 bps, and the estimate rose to 400,717 bps. Both children then +held about 30 fps through the remaining run; the healthy original delivered +2,241 frames with no assembly loss, while the constrained child delivered 1,941 +frames with 25 assembly gaps during congestion. The sampled pacer queue peaked +at 57 packets and stayed at two or fewer after recovery. This demonstrates +sustained stock recovery in that run, without an algorithm change; it does not +identify the earlier missed recovery as either a long cooldown or a permanent +stall. Temporary tracing and the extended fixture duration were removed, and +all capture/test processes exited. + +A bounded CPU-affinity check retained the original 1280x720 child and two +compatible 640x360 consumers through fifteen seconds with only the capture +subprocess restricted to one core, then restored its original affinity. All +three received about 30 fps with the same shared group and decoder process. +The subprocess used only about 11.5% of that core during the restriction, so +this did not induce overload and cannot establish resource-adaptation parity. +The temporary fixture was removed and process/affinity cleanup verified. + +A later 46-second VP8 check used the actual `AdaptiveEncoder` and mailbox at +2560x1440, 60 fps and 12 Mbps, with an independent raw-input producer. An +ignored source copy added only `GetStats()` observations. The eight-second +baseline already averaged about 33 encoded fps and reduced to 1920x1080 for +bandwidth/quality. Restricting the already initialized process to one core for +18 seconds used about 88% of that core but delivered only about 0.24 fps; +average encode time reached 1.4-3.8 seconds. Restoring affinity recovered about +60 fps at 1280x720 during the final 20 seconds. Stock statistics recorded two +quality adaptations and no CPU adaptations. This exposes a severe overload +limit, not CPU-adaptation parity. The helper reported affinity restoration and +normal completion; its PowerShell wrapper failed to retain an exit-code handle, +so an OS exit-code pass is not claimed. No product code was changed. + +The pinned [CPU detector][cpu-detector] explains why large encode times do not +necessarily produce a CPU adaptation. Its default software thresholds are +85/42%, with 120 frame samples, three initial checks and two consecutive high +checks; checks run every five seconds after an initial 100 ms check. A change +in pixel count or a gap above 1,500 ms between frames entering encoding resets +the samples and initial-check count. Before enough samples exist, usage returns +the threshold midpoint rounded to 64%, matching the repeated 64% observations +after the multi-second stalls. The [resource manager][resources] is connected: +built-in VP8 retains the [encoder default][encoder-default] CPU opt-in and is a +software encoder. VSE invokes `OnEncodeStarted` on its encoder queue just +before the codec call; raw arrival and `OnDiscardedFrame` do not supply CPU +samples. Replaying dropped inputs as encoded work would misrepresent feedback. + +Screener's synchronous drain and latest-input mailbox reduce arrivals into VSE +under this load, but the [same pinned VSE][vse] also skips queued frames before +calling the CPU observer. Therefore moving admission or reporting source drops +alone is not an established fix for multi-second encode gaps. Chromium's pinned +Call also [caches system core count][cpu-count]; changing process affinity after +encoder initialization is an extreme constraint, not a measured one-core-device +or Chrome comparison. The 0.24 fps result belongs only to this helper. Keep +overload acceptance open without changing stock thresholds or adding a custom +CPU controller. + +## Replacement And Preservation Map + +| Current owner | Treatment if a pool-backed engine is proved | +| --- | --- | +| Go room/controller, embedded STUN, SFU service lifecycle | Preserve. A codec experiment does not alter authority, NAT policy or single-operation routing. | +| LiveKit forwarding/BWE/projection and Pion transport | Keep for existing encoded forwarding. Replace a Native endpoint owner only if the new library actually assumes that responsibility; never run two controllers on one edge. | +| Native output construction, `Source.BeginFrame`, codec workers | Review together against the replacement's capabilities. Remove superseded activation, rate and switching policy in the same integration, not through a permanent legacy path. | +| Native profile handoff, source clock, bounded queues and retirement | Preserve their verified behavior even if their implementation moves. Do not lose the profile-install acknowledgement or reintroduce double FPS limiting. | +| Client sessions, Global Link, UI, diagnostics and packaging | Outside an encoder-policy replacement. Preserve the separately useful fixes/enhancements recorded in TODO and Git. | + +An engine migration must also retain direct Native SFU publication, independent +Native room sessions, native ICE improvements and mixed Browser/Client relay. +Changing transport libraries does not automatically preserve these APIs. No +unrelated page rewrite or platform-adapter deletion is justified by this review. + +## Integration Gates + +1. Reuse the now-executed native VSE/VP8 boundary; do not repeat a mock-only + factory probe or fetch a second full engine to show buffer reuse. Validate + hardware H264 and physical-resource feedback with the existing capture adapter. +2. Measure source replacement, overload, real scheduling, perceptual quality, + transport overhead and actual CPU/GPU cost. Preserve correction/drop/resource + feedback with explicit owners before accepting the product integration. + The delayed-codec counterexample above means the per-sender factory cache + cannot be selected as-is; evaluate shared complete encoding pipelines before + adding another resource/membership coordination layer. +3. Connect the separately verified encoded bypass and shared lower derivation + through that same representation owner. If preserving stock + control requires substantial VSE/source patches or a custom rate/quality + policy, report that cost instead of calling it a small factory adapter. +4. The narrow pass permits an isolated implementation candidate, not release + approval. Select the integration through an ADR refinement after resolving + the remaining owners. Preserve the current candidate; never merge it solely + to manufacture a new branch base. Follow + the stable executable paths and serialized physical workload rules in + CONTRIBUTING. The quiet-source update defect has real WGC acceptance and the + owner closed the Win10 display report; distinct game failures require fresh evidence. + +[chromium-deps]: https://chromium.googlesource.com/chromium/src/+/refs/tags/152.0.7977.82/DEPS +[chromium-hint]: https://chromium.googlesource.com/chromium/src/+/refs/tags/152.0.7977.82/third_party/blink/renderer/modules/peerconnection/media_stream_video_webrtc_sink.cc +[transport]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/call/rtp_transport_controller_send.cc +[rtp-budget]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/call/rtp_video_sender.cc +[simulcast-rates]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/modules/video_coding/utility/simulcast_rate_allocator.cc +[vse]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/video/video_stream_encoder.cc +[resources]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/video/adaptation/video_stream_encoder_resource_manager.cc +[cpu-detector]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/video/adaptation/overuse_frame_detector.cc +[encoder-default]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/api/video_codecs/video_encoder.cc +[cpu-count]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/rtc_base/cpu_info.cc +[video-adapter]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/call/adaptation/video_stream_adapter.cc +[vp8]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/modules/video_coding/codecs/vp8/libvpx_vp8_encoder.cc +[sender-hint]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/pc/rtp_sender.cc +[media-engine]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/media/engine/webrtc_video_engine.cc +[simulcast-encoder]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/media/engine/simulcast_encoder_adapter.cc +[encoder-factory]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/api/video_codecs/video_encoder_factory.h +[encoder-api]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/api/video_codecs/video_encoder.h +[encoded-image]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/api/video/encoded_image.h +[frame-metadata]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/video/frame_encode_metadata_writer.cc +[broadcaster]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/api/video/video_broadcaster.cc +[test-proxy]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/test/video_encoder_proxy_factory.h +[bitrate-adjuster]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/video/encoder_bitrate_adjuster.cc +[vp8-default-temporal]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/modules/video_coding/codecs/vp8/default_temporal_layers.cc +[vp8-decoder]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/modules/video_coding/codecs/vp8/libvpx_vp8_decoder.cc +[epic-migration]: https://github.com/EpicGames/PixelStreamingInfrastructure/blob/f826b19279ef5a8401341bc046129f1726999db2/Docs/pixel-streaming-2-migration-guide.md#L319 +[epic-settings]: https://github.com/EpicGames/PixelStreamingInfrastructure/blob/f826b19279ef5a8401341bc046129f1726999db2/Frontend/Docs/Settings%20Panel.md +[webrtc-build]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/docs/native-code/development/README.md +[codec-initializer]: https://webrtc.googlesource.com/src/+/6f37672d358475cd17544121a12494da454d85fb/modules/video_coding/video_codec_initializer.cc +[chromium-mf]: https://chromium.googlesource.com/chromium/src/+/refs/tags/152.0.7977.82/media/gpu/windows/media_foundation_video_encode_accelerator_win.cc diff --git a/docs/status.md b/docs/status.md index 45d05ae..dc56d4b 100644 --- a/docs/status.md +++ b/docs/status.md @@ -1,18 +1,23 @@ # Current Status -Last updated: 2026-09-06 +Last updated: 2026-09-09 This is the current execution index. Git history owns completed timelines; [verification status](./verification-status.md) owns evidence boundaries. -## Last Production Postflight +## Recorded Pre-Cutover Production Check -- The pre-Go postflight at `https://share.bonfire.icu` verified the strict - `screener-v19` Browser/server contract. Current revision, release, artifact, +Read-only verification on 2026-09-08 confirmed the pre-cutover production state +below. The runtime release descriptor and deployment record own later cutover +identity; this historical check is not a live service-status endpoint. + +- The running Go application and public Browser asset at + `https://share.bonfire.icu` retain the strict `screener-v21` contract. + Current revision, release, artifact, manifest, and asset identity are retained by the immutable release descriptor, runtime `REVISION`, and deployment record rather than copied into this source snapshot. -- The latest scoped postflight found public health and the immutable Browser +- The latest scoped check found application health and the immutable Browser asset available, with Screener, LiveKit, coturn, and nginx active and without restarts. - Production enables SQLite room authority at @@ -51,151 +56,103 @@ This is the current execution index. Git history owns completed timelines; ## Current Source -- Source uses the strict `screener-v21` Browser/server contract and one - shared Go server core for Hosted and Client; the graph controller is universal - and the Host-star configuration and wire path are removed. Optional SQLite - stable authority, non-expiring - local preferred code, atomic room replacement and the Host codec selector - remain implemented; production selects stable storage. Graceful restart, crash, - timeout and network loss use one reconnect state. LiveKit room teardown cannot - stop Host-owned capture. -- One event-driven controller owns the committed graph and one room-serial child - operation. Initial direct acquisition uses a five-second foreground window; - exact transport-connected progress may retain that candidate through the - total deadline, while first decoded frame remains the only commit proof. SFU - provides working media before finite background direct convergence. Healthy - decoded edges remain sticky. A connected Viewer's advertised effective - capacity survives sharing-generation graph replacement and is cleared only - when that Viewer departs or the room is deleted. A replacement Host identity - resets physical route ownership without discarding Viewer capacity. Retiring - SFU roots remain only as inactive anchors while Peer descendants reassign. -- WebRTC/LiveKit own media adaptation. After availability work, one persistently - degraded native edge may use the existing serial operation. A P2P candidate - needs first-frame readiness, fresh native healthy sender evidence, and three - clean receive windows that do not regress resolution or rounded FPS. A zero- - frame incumbent is zero delivery only against an overlapping candidate that - decodes video. Peer candidates stay first; bounded SFU remains only the whole- - Host-root suffix; when no different Peer is eligible, the same operation may - append one same-parent connection-regeneration candidate. Inconclusive current- - sender windows retain the bounded Peer operation; failed exact candidates stay - consumed until the sender or candidate opportunity changes. The pre-share gate - defaults on and locks while sharing. -- NAT prediction stays authority-gated and defaults on only when offered. Browser - and Native P2P use one connection-local rule with Site or Public-Link survey - endpoints; pure LAN, route scoring, and SFU preference remain unchanged. - Enabled acquisition now shares a three-attempt budget in the route controller, - across foreground P2P and background direct continuation. Viewer progress - reflects actual candidate creation; quality trials retain their prior budget. - Candidate and selected-path logs retain anonymous `ordinary | predicted | - unknown` provenance. Ordinary candidates and SFU fallback remain unchanged. -- Every direct, Browser-relay, and Host SFU video sender owns one clone of its - source track; the original remains presentation and source authority only. - Replacement, rollback, unpublish, physical LiveKit sender recreation, and - teardown retire that clone. Sibling clones may still share source pressure. -- Screener has no custom SFU layer list, quality score, layer selector, - all-pairs probe, parent-wide prediction or periodic rebalancing. P2P and SFU - still recover their current route before actual failure enters reassignment. -- Viewer hidden/freeze/pagehide suppress decoded-stall routing authority; - visible/resume/pageshow rebaseline and rearm current-frame proof. Host and - Viewer SFU clients disable LiveKit page-leave auto-disconnect, so SFU recovery - cannot stop Host-owned capture. Native Viewer controls still own playback, - and manual reconnect stays on the current route. A Client-launched Viewer can - receive H.264/VP8 and Opus through a native encoded source, bridge one local Browser - preview, and reuse that source for compatible P2P children; unsupported codec - or failed bridge returns to the Browser peer. -- Viewer presentation derives access, Host, route, playback, and runtime state - from separate owners. An exact media generation plus monotonic proof epoch - owns visible-frame truth; every invalidation rearms proof, while stale epochs - and replaced generations cannot clear or revive the current result. Viewer - diagnostics continue through incomplete presentation windows, but those - windows remain ineligible for route-quality convergence. -- Host, Viewer, join, and access surfaces share the living-room presentation - model. Room codes, invitation URLs, the current display name, selected - participant details, route labels, and metrics remain visible in visual mode; - Host and Viewer couch/topology identities stay consistent across expression - modes, and bounded deep trees scroll within their own surface. +- The candidate uses one shared Go core and the strict `screener-v22` + Browser/server contract, Native control v9 and capture v7. These artifacts + form one private contract. Optional SQLite persists stable room authority; + participants, routes and media remain process-only. +- Hosted Screener owns configured Binding-only STUN and optional SFU UDP + listeners in-process. SFU SDP/ICE and direct-subscriber demand use authenticated + room signaling. There is no external SFU room service, Browser LiveKit SDK, + media token or second signaling connection. Startup and shutdown own all + configured listeners; Local and public-link Clients remain P2P-only. +- One controller still owns the committed graph and one serial child operation. + Endpoint copy limits, first-decoded-frame commits, bounded direct acquisition + and candidate-relative quality proof remain in + [routing and transport](./product/routing-transport.md). Exact physical + subscription/publication closure now releases the matching SFU reservation. +- Native parents and embedded SFU share a Pion/LiveKit media adapter for + forwarding, bandwidth estimation, allocation, pacing and recovery. A Native + parent reuses a suitable encoded output and derives a missing lower output + only on direct-child demand. Source groups retain the highest needed output + and lower fallbacks; each child receives one selected output. Native Host SFU + publication reuses the source directly and has one aggregate upstream budget. + [Media quality](./product/media-quality.md) owns this behavior and its limits. +- Browser senders retain source-owned clones and stock WebRTC adaptation. + Current SFU demand updates the publisher's active output prefix; live quality + changes and ICE recovery retain their connection identities. Healthy media + survives transient room-signaling loss, and transport teardown cannot stop + the Host's original capture track. +- Host tab authority is separate from the persistent origin resume hint. The + Client admits two independent native control sessions; reconnect, room + replacement and source retirement remain fenced by their current owners. + Public-link startup reserves its local listener before starting a tunnel. +- Viewer presentation retains current-media identity and decoded-frame proof + across lifecycle changes. Diagnostic capture, signaling and native-request + events are opt-in and bounded; [configuration](./reference/configuration.md) + owns activation and export. Diagnostics do not add media or routing authority. +- The runtime-only container recipe reuses the Server release rather than + rebuilding the application. Non-root/read-only execution, memory/SQLite + restart behavior, diagnostic export, STUN and UDP lifecycle pass in the local + Linux VM. Public-network media and production cutover remain separate. ## Source/Production Relationship -- The deployment record owns exact source/release comparison. Routine releases - change this file only when its semantic or operational snapshot changes. +The source change requires a coordinated production cutover. The recorded check +above retains the external-service history; exact release identity belongs to +the deployment record. The +[self-hosting transaction](./operations/self-hosting.md#coordinated-embedded-media-cutover) +replaces those services and their configuration together. The routine application +wrapper requires an existing embedded Go deployment and cannot perform or roll +back the initial infrastructure change. ## Active Boundaries -- Desktop Host background capture remains unconfirmed. Current Browser screening - did not reproduce a drop; Viewer lifecycle work prevents frozen JavaScript - wall time from becoming route-failure evidence but is not capture keepalive. -- Current-path quality can drive local convergence in source and production, - including bounded same-edge connection regeneration. Both default it on with a - pre-share Host opt-out. Weighted/global optimization remains parked. -- Source contains the cross-platform Client outside the current Web release. - One Go entry starts a process-level loopback capability service, then opens the - current system-Browser launcher for Local, temporary public-link, or saved-Site - operation. The saved Site remains allowed while another room source runs; the - same Host UI explicitly selects - Browser capture or an exact native screen/window. Windows Native now shares - the VP8/Auto/H264 selector, with libvpx software VP8 and hardware H264; Auto - uses a bounded target-profile throughput check. Both VP8 and Auto-selected - H264 passed real Browser delivery and codec-stable live/source changes. - Windows gates prove - packaged Local/Site operation, WGC hardware-H.264 plus process/system audio, bounded - shared-encode P2P, and cleanup. The window arm also proves source end and - same-room reselection; display-source lifecycle remains a separate physical - gate. An explicit target assembler also produces a - Linux amd64 package with proved Local/one-link runtime and a structurally - verified macOS arm64 package. Its capture sidecar passes an arm64 compile, - hardware-H.264 IDR self-test, and audio-adapter build, but the package and - capture path remain unrun on a physical Mac. The Linux package now includes a - thin Portal/PipeWire/GStreamer hardware-H.264 capture adapter and passes its - compile, probe, package, startup, and shutdown checks; real desktop capture, - system audio, and recovery remain unproved. One-link mode preserves the Host - Local authority while exposing its ordinary invitation and HTTP/WebSocket - control path through a session-scoped Quick Tunnel. An independent Linux Pion - Viewer has received that native media over a direct ICE pair; remote Browser - media remains unproved. Site and one-link native shares also make one bounded - best-effort PCP/UPnP/NAT-PMP mapping and Pion Universal-UDP-mux STUN discovery - on their media socket. A public-link run delivered 35 H.264 packets to an - independent Linux Viewer; no predicted-path win is yet claimed. - Native P2P edges now negotiate TWCC and send Pion GCC's target-versus-source - payload category through the existing sender-quality evidence path without - pacing or changing the shared encode. The existing quality operation may test - a Browser sender for one persistently degraded Native edge without lowering - healthy shared edges. A local bridge supplies native Host media to the existing - Browser LiveKit publisher; an isolated LiveKit gate - proves default 1080p delivery, a live change to 480p, and complete cleanup - without a second SFU client. - A Windows gate proves Browser Host to Client-native Viewer delivery and Chrome - decode; an RTP gate covers bounded H.264/Opus child relay and cleanup. - Client-assisted Browser H.264 capture now reuses one local sender for Native - fanout while quality convergence is enabled. Its two-Viewer gate covers - cadence, live controls and Client-exit recovery; weak-network acceptance - remains open. Native RTP forwarding preserves padding and its sequence - continuity without requiring decoded dimensions for capacity evidence. - Native control treats malformed/stale/repeated per-edge signaling as disposable; - unexpected Client close reaches the current Browser participant through the - existing fence. Windows - retains one converted frame for quiet-source keyframe recovery. Its live - profile gate keeps two PeerConnections while moving the hardware source from - 720p30 to 1440p60, then to 480p15 while paused, and resumes both Viewers; - current source-switch, static, and crash gates pass, while cross-version - capture remains open. - Local Client access is open by default and can be protected with a user-chosen - password from the Browser launcher; Hosted Site access is unchanged. - Non-Windows native media remains outside physical acceptance. The default - packaged Client launcher performs a non-blocking official GitHub Release check; - the deployment tree provides a read-only operator check against the same full- - SHA release identity. Neither installs or interrupts a running share. +- Native shared-output implementation follows the + [encoder-pool review](./research/webrtc-encoder-pool.md); remaining release + acceptance is held in TODO. Windows capture and derivation now attach stock + WebRTC output pipelines; both codecs pass bounded process-level shrinking and + recovery checks. Incompatible weak-consumer grouping now has real relay + split/rejoin and independent-dimension delivery evidence; exact throughput, + representative-network behavior and extreme-overload limits remain explicit. + The extreme single-core check reaches WebRTC's sample-reset boundary; it does + not justify another application resource controller. + Independent Client/service fixes remain preserved. +- Browser and Native H264/VP8 publication through embedded SFU pass Browser decoding, + live-profile and cleanup checks; the Native path also delivers Opus. Bounded + Native VP8 forwarding/derivation and stopped-upper recovery have synthetic + codec and shaped-network evidence. Hardware + overload, below-lowest-output, public-network and packaged acceptance remain open. +- Current Windows VP8 and H264 owned-window minimize/profile/restore checks pass, + including live and paused changes and the H264 source-switch/restart path. + The owner confirmed Windows 10 display sharing resolved. Distinct game + HWND/device-loss behavior, 60 fps endurance and hardware overload remain + separate evidence boundaries. The reported + machine freeze has no confirmed cause; [TODO](./todo.md) owns the current + authorization and serialization of physical workloads. +- Linux capture producers compile. Updated macOS producers still require an + SDK build; neither platform's current capture/audio/recovery has physical + acceptance. Prior package and single-output results do not prove the new + multi-output contract. The owner defers these secondary-platform matrices; + Windows Client and Browser remain this phase's acceptance targets. Two Native + control sessions and rooms now pass concurrent media and independent retirement. +- Matched platform artifacts, + representative packet loss/congestion, 20-Viewer endurance, public-link + Browser delivery and restricted-NAT success remain in the + [TODO ledger](./todo.md) and [verification status](./verification-status.md). + Browser/OS suspension remains a physical limit, not a keepalive promise. +- Windows Native Host, Native embedded-SFU and Browser-to-Client fanout pass the + current combined profile, pause/recovery and closure checks. Pure Browser + reuse was then tested separately: late-join recovery works, but legacy fanout + fails independent quality and sender accounting; standard transforms reject + cross-source frames. The experiment adds no product path. Normal Browser + senders and optional Client fanout remain the supported composition. ## Release Acceptance -Client acceptance remains open for the reported Windows 10 display startup and -game-specific source behavior. Windows 11 display, minimized-source recovery, -live presets and source-switch checks pass, but do not establish those reports -as resolved. NAT acquisition has controller and signaling coverage, not a new -public-network success-rate claim. The candidate removes the Host-star -configuration and wire shape under `screener-v21` and moves the server to Go. -The authorized cutover needs matching Web/Client builds, the updated unit file -that replaces `NODE_ENV` with `SCREENER_ENV` before the first Go release, -an active-session check, and configuration restoration on rollback. The actual -deployment result belongs to its release record; [TODO](./todo.md) owns -remaining device/network acceptance. +The phase requires matching current Web/Server/Client/capture artifacts and the +remaining media acceptance before one owner-accepted integration. The first +production move also needs an active-session check, accepted share interruption, +released old UDP listeners, and verified unit/environment/proxy/service recovery. +Restoring only an application symlink cannot roll back that protocol and +infrastructure transaction. No deployment or current-platform acceptance is +claimed by these source documents. diff --git a/docs/todo.md b/docs/todo.md index 6bab870..4efe0d5 100644 --- a/docs/todo.md +++ b/docs/todo.md @@ -1,153 +1,121 @@ # Current TODO Ledger -Last reviewed: 2026-09-06 +Last reviewed: 2026-09-09 -Only **Now** is executable. Observations, old branches, experiments, and parked -topics are not implementation authority. +Only **Now** is executable. Product modules own behavior, research owns evidence, +and Git/PRs own completed history. A parked idea is not implementation authority. ## Now -The returned Go core and frontend fixes are reconciled on the integration -candidate. The owner authorized hygiene cleanup, matched Server/Client builds, -one squash PR and a controlled deployment. Complete the existing checks and -first-Go unit/environment rollback before cutover. Device/network checks below -remain evidence gaps, not claims established by compilation or local loopback. +Owner-authorized phase: [embedded STUN/SFU and node-local media](./adr/0013-embedded-node-local-media.md). +Finish this phase as one integration, preserving the completed mainline checkpoint. -1. **Client physical acceptance.** Physically validate macOS and Linux - capture/audio/recovery, a decoded public-link Browser Viewer, and a NAT case - rescued beyond ordinary STUN, plus one mixed Browser/Native Viewer relay. -2. **Client owner acceptance.** Review the localized terminal, Browser capture - through Native fanout, live controls and Client-exit recovery before merging. - Confirm weak-path convergence with effective media throttling; a DevTools - command succeeding without limiting RTP is not a weak-path pass. -3. **Capture startup and controls.** Diagnose Windows 10 selecting an entire - display failing to start while window capture succeeds; do not conflate it - with exclusive-fullscreen game capture. Current Windows 11 checks cover - display startup, minimized-source recovery and live presets. Verify remaining - game HWND/device-loss behavior and physically compare the implemented Windows - display-aspect correction with the reported stretched game, including GPU - scaling not exposed by the active display path. Compare bitrate changes against - same-content delivery before changing encoder settings. -4. **NAT acquisition and release acceptance.** Validate the implemented bounded - three-attempt owner on restricted Browser/Native pairs; a new connection does - not guarantee a different NAT mapping. The added prepare progress field is - incompatible with old strict pages. The owner authorized the coordinated - Web/Server/Client update; check active sessions before the cutover and - retain the previous unit/environment alongside the application rollback; - do not mix the new field into an ongoing old-version share. -5. **Native codec acceptance.** Windows Native VP8 and Auto-selected H264 now - pass real Browser playback, live presets, paused changes and source-switch - checks while preserving the selected codec. Validate GPU-heavy Auto selection - and the tight four-second probe/five-second startup budget on other hardware; - do not claim throughput screening guarantees perceptual quality or later - load. Other native platform capture encoders remain H264. +1. **Release integration.** One complete PR must contain the Native/embedded + media work, retained Client fixes, final corrections, current documentation + and research outcomes. Build matching Web/Server/Client/capture artifacts + from the accepted revision; main remains a low-frequency integration branch. +2. **Coordinated production cutover.** Recheck active sessions, preserve the + exact old release/unit/environment/proxy/media-service state and SQLite + recovery material, then replace external LiveKit/coturn with the embedded + process. Verify HTTP/assets, STUN, SFU media, listener retirement and rollback + using [self-hosting operations](./operations/self-hosting.md). + The routine application updater alone cannot perform this first cutover. +3. **Matched delivery.** Provide the current Windows Client package and Server + descriptor/archive with matching hashes and notices. Production pages and + old Clients cannot be mixed across the private protocol change. Formal + public GitHub Release/image publication remains a separate explicit action. + +Current evidence and limits: + +- Windows/Browser are the primary targets. Native Host, Native embedded SFU and + Browser-to-Client fanout pass profile, audio, pause/recovery and cleanup checks. + Local container checks cover non-root/read-only operation, memory/SQLite, + diagnostics, STUN and media listener lifecycle; they do not prove target-network + media. macOS/Linux physical matrices may follow user feedback. +- Compatible Native consumers share one complete encoding/adaptation pipeline; + incompatible direct-child demands remain independent. Reuse suitable received + encodings and derive missing outputs locally. No ancestor-cache discovery, + extra upstream layer subscription, custom BWE or resource controller is accepted. +- Extended weak-network recovery passes. Preserve the earlier missed twelve-second + observation and extreme single-core sampling-reset limitation in + [encoder research](./research/webrtc-encoder-pool.md); neither establishes a + universal recovery deadline or a missing callback. +- The owner closed Win10 whole-display sharing. Quiet-source settings pass real + WGC/Browser acceptance. Reopen distinct game/display failures only from new + evidence, not a failed automation focus precondition. +- [Browser encoded reuse](./research/advanced-peer-distribution.md#september-9-results-and-decision) + completed after the mainline checkpoint. Late join can recover with one keyframe + request, but legacy fanout fails independent quality and sender accounting. + Standard Transform rejects cross-source frames. No product adapter is accepted. + Keep normal Browser senders and the verified optional Client fanout. +- Run one bounded capture/codec workload at a time, with cleanup before the next. + Keep VM workloads out of that interval and use stable executable build paths. ## Parked Product Work -1. **Broader quality optimization.** Same-edge connection regeneration is now - part of the bounded local convergence path. Weighted/global optimization, - parent-wide prediction, startup-limited inference, complete cross-clone - source isolation, and alternative SFU policies remain unaccepted. Reopen only - from exact path evidence; do not add a score, all-pairs probe, periodic - rebalancer, or ordinary SFU preference. Validate whether a quality move merely - shifts residual shared-source pressure to the new parent's siblings before - widening it. Preserve the accepted profiles, bitrate ceilings, endpoint - capacity, degradation preferences and LiveKit representations unless a - measured gain is large enough to justify the quality, resilience, - compatibility and maintenance cost through a new accepted decision. -2. **Startup codec budget.** Reconcile the bounded Auto codec probe with the - first-frame target from measured Browser evidence; do not silently weaken - Auto or move an unproved result into the availability path. Validate whether - H.264 qualification can choose a slower VP8 fallback under mixed GPU/CPU load - before adding comparative or dual-codec probing. -3. **Browser encoded reuse.** Pure Browser operation keeps normal WebRTC - senders; users can use Client to reduce repeated encoding. Revisit Browser - reuse only for a small mature adapter that preserves adaptation and media - quality without an application-owned transport or layer controller. -4. **Control-plane resource fairness.** Calibrate one deployment-wide LiveKit - control queue and simple HTTP/WebSocket ingress budgets before implementation; - prove a self-hosted stale-token revocation boundary before independently - releasing subscription egress; do not introduce accounts, polling, risk - scoring, or an independent policy framework. -5. **Client ownership.** Extract media-session owners from the Host/Viewer pages - only alongside reachable behavior work. File length alone does not authorize - a page rewrite; Host diagnostic presentation is already coalesced without - delaying media evidence or route control. -6. **C=3 structural intent retention.** Revisit multiple simultaneous one-shot - Host-root convergence intents only with an exact C=3 reproduction; do not add - periodic balancing or another topology queue for the default path. -7. **Mobile Viewer lifecycle.** Run Android Chrome and iOS Safari matrices for - autoplay, background audio, foreground recovery, lock/page reclamation, - rotation, network migration, and relay survival. -8. **Representative network acceptance.** Complete public-network direct, - Browser relay, SFU, recovery, Pause/Resume, screen-audio, real-game A/V, - two-room SFU, 20-Viewer endurance, and all-UDP-blocked bounded failure. - Validate LiveKit Server 1.13.6 simulcast retransmission under controlled loss - and correlate any remaining SFU freezes with publisher encoding counts before - changing Dynacast or representation policy. -9. **Client-input security review.** Audit HTTP/WebSocket schema, auth, - authorization, rate/body/resource bounds, errors, logs, and secrets without - adding a parallel security framework. -10. **Public-server package.** Package the exact Web/signaling, STUN/SFU, proxy, - secrets, health, and recovery contract for a user-owned server. -11. **Native media endurance.** Exercise long-lived native P2P and Browser- - mediated SFU recovery after the platform capture boundary passes. Preserve - Browser/server authority and the existing LiveKit representation policy; do - not create a second room or media policy. -12. **Platform output.** System/tab mirroring needs no product adapter, while - Remote Playback, default Cast and AirPlay do not provide a portable live - `MediaStream` output contract. Reopen only for a registered custom receiver - acting as an ordinary Viewer after a named Browser and physical receiver - pass the [platform-output gate](./research/platform-output.md). -13. **NAT inference and predictive candidates.** The deployment-gated, - connection-local capability is accepted under - [ADR-0009](./adr/0009-optional-nat-prediction.md). Production enables its - self-hosted field rollout; the general configuration default stays off. - Keep injected candidates additive and diagnostic; do not make them a - participant-wide NAT label, hard candidate skip, route score, or SFU - preference. Validate selected predicted paths and bounded resource impact - plus one selected Native mapped-port path that fails with STUN alone before - recommending either as a default for other deployments. Measure benefit of - the implemented three-attempt budget before increasing it or claiming that - independent connection attempts are independent NAT mappings. -14. **Unresolved route-state ownership claims.** Reopen active SFU failure - during an unrelated prepare and - multi-child relay-evidence ownership only from an exact current-wire - reproduction. Do not add a second revision namespace, parallel failure - state, or generalized evidence map from static possibility alone. -15. **Unified release and update surface.** Build Client packages, Server - runtime, future OCI images, and deployment bundles from one intentionally - triggered immutable application release identified by its full commit SHA. - A validated `main` push now produces that application release. An explicit - `client_checks=true` workflow dispatch produces the three native-runner - Client candidates with SHA-256 metadata. The default Client - launcher now performs a non-blocking GitHub Releases check and the tracked - deployment tree provides an operator-invoked read-only Server check. Formal - GitHub Release publication, future OCI images, and deployment bundles remain - explicit distribution work; do not add automatic install, container - self-update, Watchtower, compatibility ranges, or active-share interruption - before distribution and rollback evidence requires them. -16. **Go server consolidation.** Done in the candidate under - [ADR-0012](./adr/0012-shared-go-backend-core.md): Hosted and Client share one - Go core and the Node server, bundled runtime, and supervisor are deleted. Do - not create or retain two room, signaling, persistence, or route-controller - implementations. Package size, startup, and idle memory are measured in the - [consolidation research](./research/server-consolidation.md). Remaining - acceptance: rerun the physical Client and Browser-contract gates on this - build, and sequence the deployment cutover, installing the updated unit, - which replaces `NODE_ENV=production` with `SCREENER_ENV=production`, before - the first Go release, then proving the release wrapper on a host without - Node. -17. **Opt-in diagnostics.** Replace ad hoc console logging with an explicit - Client/Server debug option and bounded diagnostic export. Include revision, - runtime state and sanitized capture/connection events; exclude credentials, - media and raw process memory by default. A compressed feedback bundle needs - deliberate collection and retention, not automatic uploads. -18. **Remaining audit behavior.** During the owning refactor, reproduce and - resolve optional Local-password cookies across HTTP LAN/HTTPS public origins; - audio-process failure without stopping healthy video; natural capture EOF - attribution; and Native prepared-bridge failure that must disable Native - for the current Viewer session before retrying. Storage I/O failure needs - an authority-consistent recovery policy, not catch-and-continue guards. - The Local duplicate-launch guard is the in-process listener bind itself; - it refuses a second Client on the same port and hands nothing over. +1. **Comprehensive Debug feedback.** Authorized as follow-up on 2026-09-09 for + Client and Server. Use mature projects' feedback-bundle practice. A first + report should contain enough context to diagnose without another build just + to add missing logging: operation start/end and identity, requested/applied + settings, capture/codec/ICE/transport transitions, resource and queue pressure, + and precise failed checks with safe expected/actual values. + Debug-on completeness takes priority over minimal log volume; keep explicit + resource bounds without silently removing essential context. Exclude + credentials, access/media tokens and private keys. Do not promise complete + anonymization; document residual identifiers, network/system information and + export contents clearly. Keep local opt-in collection and user-controlled + sharing. Design one observability surface, not incident-specific patches. + Assess stack and memory snapshots by diagnostic value and document their + contents; do not silently include raw media or secret-bearing memory. + Existing exports are owned by [configuration](./reference/configuration.md). +2. **Public distribution and updates.** Candidate packagers, immutable descriptors, + notices, the runtime-only OCI recipe and release checks exist. Choose the + public asset set and release notes before publishing a full-SHA GitHub Release + or image. Do not add automatic installation, container self-update, Watchtower, + compatibility ranges or active-share interruption without distribution and + recovery evidence. Ordinary branch pushes must not run expensive CI packaging. +3. **Representative device/network acceptance.** Exercise public-network direct + and relay paths, two-room SFU, real-game A/V, twenty-Viewer endurance, + sustained loss/recovery and all-UDP-blocked bounded failure. Correlate freezes + with publisher/receiver evidence before changing representation policy. + Physical Android/iOS work includes autoplay, background/lock, rotation, + network migration and relay survival. Secondary desktop platform matrices + remain explicitly deferred; compilation does not prove capture. +4. **Browser encoded-input API.** Reopen reuse only when a supported API supplies + usable injection, feedback and timing ownership. Chromium's experimental + RTCEncodedSource is a lead, not current M152 availability or automatic + adaptation. An adapter must pass independent quality, recovery and truthful + statistics before product integration. +5. **Broader quality work.** Reopen from measured benefit at acceptable complexity. + Preserve chosen profiles, bitrate ceilings, endpoint capacity and P2P-first + routing unless a new accepted decision supports changing them. No weighted + score, all-pairs probes, periodic rebalancing, parent-wide prediction or + room-wide minimum. Check whether a quality move merely shifts pressure to + another parent's siblings before widening policy. +6. **Startup codec budget.** Compare bounded Auto and first-frame behavior on + representative hardware. Throughput qualification does not guarantee perceptual + quality or later GPU load. Do not silently weaken Auto or introduce + comparative/dual probing without evidence. +7. **NAT field evidence.** Preserve [ADR-0009](./adr/0009-optional-nat-prediction.md). + Validate selected predicted/mapped paths and bounded resource cost, including + a Native case rescued beyond ordinary STUN. Measure the three-attempt budget + before expanding it; new connections do not guarantee independent mappings. + Candidates remain connection-local and diagnostic, not participant labels, + hard skips, route scores or SFU preference. +8. **Control/resource fairness and input review.** Reassess the authenticated + WebSocket/SFU owner, HTTP/body/resource bounds, authorization and error/log + handling before adding a queue or limiter. No parallel security framework, + accounts, risk score or speculative policy layer. +9. **Reachable ownership/refactor work.** Extract page media-session owners only + alongside behavior changes; file size alone is not a rewrite reason. Reopen + C=3 structural-intent retention, SFU failure during unrelated prepare and + multi-child evidence ownership only with current-contract reproductions. + No new revision namespace, failure-state mirror or topology queue by default. +10. **Storage fault recovery.** Ordinary transactions persist before memory + changes; failures must preserve authority. Message-handler panics must unwind + locks. Damaged-disk/COMMIT/ROLLBACK recovery is not established; choose its + policy explicitly rather than adding catch-and-continue or retries. +11. **Platform output.** System/tab mirroring needs no adapter. Remote Playback, + Cast and AirPlay do not provide a portable live MediaStream receiver. + Reopen for a registered receiver acting as an ordinary Viewer after the + [platform-output gate](./research/platform-output.md) passes. diff --git a/docs/verification-status.md b/docs/verification-status.md index d549674..98691f2 100644 --- a/docs/verification-status.md +++ b/docs/verification-status.md @@ -176,8 +176,9 @@ The current Windows Native candidate additionally passed manual VP8 and Auto- selected H264 playback, with actual Host/Viewer RTP codec checks across live presets, paused changes and source replacement. A three-attempt NAT acquisition has deterministic controller/signaling coverage, not a field success-rate -claim. The completed checks do not resolve the reported Windows 10 monitor -startup or game-specific aspect/window-replacement issues. The matching bundle +claim. Windows 10 monitor sharing was confirmed resolved by the owner on +2026-09-09. Game-specific aspect/window-replacement behavior is not established +by those checks. The matching bundle must be accepted before any coordinated private-wire deployment. - Configuration, unit tests, loopback, and synthetic signaling prove invariants, diff --git a/go.mod b/go.mod index afbed38..b893f5c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/TNTcraftHIM/Screener -go 1.25.0 +go 1.26 require ( charm.land/bubbletea/v2 v2.0.9 @@ -9,6 +9,9 @@ require ( github.com/charmbracelet/x/ansi v0.11.8 github.com/charmbracelet/x/term v0.2.2 github.com/coder/websocket v1.8.15 + github.com/livekit/livekit-server v1.13.6 + github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095 + github.com/livekit/protocol v1.50.5-0.20260821144838-17c16cf496fd github.com/netbirdio/go-nat v0.0.0-20260821095157-6b2c8c5c74e8 github.com/pion/ice/v4 v4.4.0 github.com/pion/interceptor v0.1.47 @@ -17,47 +20,108 @@ require ( github.com/pion/rtp v1.10.5 github.com/pion/sdp/v3 v3.0.19 github.com/pion/stun/v3 v3.1.6 + github.com/pion/transport/v4 v4.1.0 + github.com/pion/turn/v5 v5.0.12 github.com/pion/webrtc/v4 v4.2.18 github.com/thesyncim/gopus v0.1.1 - golang.org/x/crypto v0.48.0 + golang.org/x/crypto v0.54.0 golang.org/x/text v0.41.0 modernc.org/sqlite v1.58.0 ) require ( + buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260709200747-435963d16310.1 // indirect + buf.build/go/protovalidate v1.2.0 // indirect + buf.build/go/protoyaml v0.7.0 // indirect + cel.dev/expr v0.25.2 // indirect + github.com/antlr4-go/antlr/v4 v4.13.1 // indirect + github.com/benbjohnson/clock v1.3.5 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/charmbracelet/ultraviolet v0.0.0-20260811164956-006e29f97886 // indirect github.com/charmbracelet/x/termios v0.1.1 // indirect github.com/charmbracelet/x/windows v0.2.2 // indirect github.com/clipperhouse/displaywidth v0.11.0 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/dennwc/iters v1.2.2 // indirect github.com/dustin/go-humanize v1.0.1 // indirect + github.com/elliotchance/orderedmap/v3 v3.1.1 // indirect + github.com/florianl/go-tc v0.4.8 // indirect + github.com/frostbyte73/core v0.1.1 // indirect + github.com/fsnotify/fsnotify v1.10.1 // indirect + github.com/gammazero/deque v1.2.1 // indirect + github.com/go-logr/logr v1.4.4 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang-jwt/jwt/v5 v5.3.1 // indirect + github.com/google/cel-go v0.30.0 // indirect + github.com/google/go-cmp v0.7.0 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/uuid v1.6.0 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.8 // indirect github.com/huin/goupnp v1.2.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect + github.com/jellydator/ttlcache/v3 v3.4.1 // indirect + github.com/josharian/native v1.1.0 // indirect + github.com/jxskiss/base62 v1.1.0 // indirect + github.com/klauspost/compress v1.19.1 // indirect + github.com/klauspost/cpuid/v2 v2.4.0 // indirect github.com/koron/go-ssdp v0.0.4 // indirect github.com/libp2p/go-netroute v0.2.1 // indirect + github.com/lithammer/shortuuid/v4 v4.2.0 // indirect + github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 // indirect + github.com/livekit/psrpc v0.7.3 // indirect + github.com/livekit/server-sdk-go/v2 v2.14.0 // indirect github.com/lucasb-eyer/go-colorful v1.4.1 // indirect + github.com/mackerelio/go-osstat v0.2.8 // indirect github.com/mattn/go-isatty v0.0.24 // indirect github.com/mattn/go-runewidth v0.0.24 // indirect + github.com/mdlayher/netlink v1.11.2 // indirect + github.com/mdlayher/socket v0.6.1 // indirect github.com/muesli/cancelreader v0.2.2 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/nats-io/nats.go v1.52.0 // indirect + github.com/nats-io/nkeys v0.4.16 // indirect + github.com/nats-io/nuid v1.0.1 // indirect github.com/ncruces/go-strftime v1.0.0 // indirect + github.com/petermattis/goid v0.0.0-20260725062400-500c67a39b75 // indirect github.com/pion/datachannel v1.6.2 // indirect github.com/pion/dtls/v3 v3.1.5 // indirect github.com/pion/mdns/v2 v2.1.0 // indirect github.com/pion/randutil v0.1.0 // indirect github.com/pion/sctp v1.11.1 // indirect - github.com/pion/srtp/v3 v3.0.12 // indirect - github.com/pion/transport/v4 v4.0.2 // indirect - github.com/pion/turn/v5 v5.0.12 // indirect + github.com/pion/srtp/v3 v3.0.13 // indirect + github.com/prometheus/client_golang v1.24.1 // indirect + github.com/prometheus/client_model v0.6.2 // indirect + github.com/prometheus/common v0.70.1 // indirect + github.com/prometheus/procfs v0.21.1 // indirect + github.com/puzpuzpuz/xsync/v4 v4.5.0 // indirect + github.com/redis/go-redis/v9 v9.22.0 // indirect github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect github.com/rivo/uniseg v0.4.7 // indirect + github.com/twitchtv/twirp v8.1.3+incompatible // indirect github.com/wlynxg/anet v0.0.5 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect - golang.org/x/net v0.50.0 // indirect + github.com/zeebo/xxh3 v1.1.0 // indirect + go.opentelemetry.io/auto/sdk v1.2.1 // indirect + go.opentelemetry.io/otel v1.45.0 // indirect + go.opentelemetry.io/otel/metric v1.45.0 // indirect + go.opentelemetry.io/otel/trace v1.45.0 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.28.0 // indirect + go.uber.org/zap/exp v0.3.0 // indirect + go.yaml.in/yaml/v3 v3.0.5 // indirect + golang.org/x/exp v0.0.0-20260727155853-b88d891fe743 // indirect + golang.org/x/net v0.57.0 // indirect golang.org/x/sync v0.22.0 // indirect golang.org/x/sys v0.47.0 // indirect - golang.org/x/time v0.14.0 // indirect + golang.org/x/time v0.15.0 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect + google.golang.org/grpc v1.83.0 // indirect + google.golang.org/protobuf v1.36.11 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect modernc.org/libc v1.75.6 // indirect modernc.org/mathutil v1.7.1 // indirect modernc.org/memory v1.12.1 // indirect diff --git a/go.sum b/go.sum index 6a7419a..7076ce1 100644 --- a/go.sum +++ b/go.sum @@ -1,9 +1,35 @@ +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260709200747-435963d16310.1 h1:fXh8CsdNpjRr8R5vFdqtIxPt/Lno2IIJlYOdZBIZn0w= +buf.build/gen/go/bufbuild/protovalidate/protocolbuffers/go v1.36.11-20260709200747-435963d16310.1/go.mod h1:tvtbpgaVXZX4g6Pn+AnzFycuRK3MOz5HJfEGeEllXYM= +buf.build/go/protovalidate v1.2.0 h1:DQVrUWkmGTBij+kOYv/x2LLxwcLaGKMdzShj1/6/3H0= +buf.build/go/protovalidate v1.2.0/go.mod h1:7rYiQEhqvAipoazpVNBBH2S2f8bjG4huMVy1V2Yofn4= +buf.build/go/protoyaml v0.7.0 h1:z4oVoFicbpPefhT7WAykxUdfp0yEQlhMQ2mCZOY5V38= +buf.build/go/protoyaml v0.7.0/go.mod h1:+a0cavd0uMvirb87xdu2ZMMmjlIQoiH/N2Ich5MGSQ0= +cel.dev/expr v0.25.2 h1:K6j46C81hXtZQfuX60cVWQFBJahKSE2gfRbNuvr5bFs= +cel.dev/expr v0.25.2/go.mod h1:hrXvqGP6G6gyx8UAHSHJ5RGk//1Oj5nXQ2NI02Nrsg4= charm.land/bubbletea/v2 v2.0.9 h1:DpJCMWKgzQK8SJv4zbKKFHAI10ymWy/evClPFk0k0f8= charm.land/bubbletea/v2 v2.0.9/go.mod h1:2SkdgoTXluXJHOUwAoRlRXF/28vklb1rFl6GcgV1/ss= charm.land/lipgloss/v2 v2.0.6 h1:EaGKeuA8FvF+v2BT5VmZd2LoYLaMZJXA5n34th8nCIQ= charm.land/lipgloss/v2 v2.0.6/go.mod h1:ipDDJNSGa1hlwDtSfW1s2/xR8Vdhbut4PXh2zEKZd0Q= +github.com/Microsoft/go-winio v0.6.2 h1:F2VQgta7ecxGYO8k3ZZz3RS8fVIXVxONVUPlNERoyfY= +github.com/Microsoft/go-winio v0.6.2/go.mod h1:yd8OoFMLzJbo9gZq8j5qaps8bJ9aShtEA8Ipt1oGCvU= +github.com/antlr4-go/antlr/v4 v4.13.1 h1:SqQKkuVZ+zWkMMNkjy5FZe5mr5WURWnlpmOuzYWrPrQ= +github.com/antlr4-go/antlr/v4 v4.13.1/go.mod h1:GKmUxMtwp6ZgGwZSva4eWPC5mS6vUAmOABFgjdkM7Nw= github.com/aymanbagabas/go-udiff v0.4.1 h1:OEIrQ8maEeDBXQDoGCbbTTXYJMYRCRO1fnodZ12Gv5o= github.com/aymanbagabas/go-udiff v0.4.1/go.mod h1:0L9PGwj20lrtmEMeyw4WKJ/TMyDtvAoK9bf2u/mNo3w= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= +github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/brianvoe/gofakeit/v6 v6.28.0 h1:Xib46XXuQfmlLS2EXRuJpqcw8St6qSZz75OUo0tgAW4= +github.com/brianvoe/gofakeit/v6 v6.28.0/go.mod h1:Xj58BMSnFqcn/fAQeSK+/PLtC5kSb7FJIq4JyGa8vEs= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= +github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= +github.com/cenkalti/backoff/v5 v5.0.3/go.mod h1:rkhZdG3JZukswDf7f0cwqPNk4K0sa+F97BxZthm/crw= +github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= +github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/charmbracelet/colorprofile v0.4.3 h1:QPa1IWkYI+AOB+fE+mg/5/4HRMZcaXex9t5KX76i20Q= github.com/charmbracelet/colorprofile v0.4.3/go.mod h1:/zT4BhpD5aGFpqQQqw7a+VtHCzu+zrQtt1zhMt9mR4Q= github.com/charmbracelet/ultraviolet v0.0.0-20260811164956-006e29f97886 h1:rdnVWKgJpTVXKuKuJyxDJ+NFJdUaUqGvyGy61OcvlbA= @@ -18,44 +44,199 @@ github.com/charmbracelet/x/termios v0.1.1 h1:o3Q2bT8eqzGnGPOYheoYS8eEleT5ZVNYNy8 github.com/charmbracelet/x/termios v0.1.1/go.mod h1:rB7fnv1TgOPOyyKRJ9o+AsTU/vK5WHJ2ivHeut/Pcwo= github.com/charmbracelet/x/windows v0.2.2 h1:IofanmuvaxnKHuV04sC0eBy/smG6kIKrWG2/jYn2GuM= github.com/charmbracelet/x/windows v0.2.2/go.mod h1:/8XtdKZzedat74NQFn0NGlGL4soHB0YQZrETF96h75k= +github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= +github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= +github.com/cilium/ebpf v0.8.1/go.mod h1:f5zLIM0FSNuAkSyLAN7X+Hy6yznlF1mNiWUMfxMtrgk= +github.com/cilium/ebpf v0.22.0 h1:v2ktp0roffpMOj2MMf3idtCQZOsAoC4BJbAJN+ke2bY= +github.com/cilium/ebpf v0.22.0/go.mod h1:CDzZbe2hC5JjlDC+CY3KFCzlYwN4gbxppYM+Z10bQt4= github.com/clipperhouse/displaywidth v0.11.0 h1:lBc6kY44VFw+TDx4I8opi/EtL9m20WSEFgwIwO+UVM8= github.com/clipperhouse/displaywidth v0.11.0/go.mod h1:bkrFNkf81G8HyVqmKGxsPufD3JhNl3dSqnGhOoSD/o0= github.com/clipperhouse/uax29/v2 v2.7.0 h1:+gs4oBZ2gPfVrKPthwbMzWZDaAFPGYK72F0NJv2v7Vk= github.com/clipperhouse/uax29/v2 v2.7.0/go.mod h1:EFJ2TJMRUaplDxHKj1qAEhCtQPW2tJSwu5BF98AuoVM= github.com/coder/websocket v1.8.15 h1:6B2JPeOGlpff2Uz6vOEH1Vzpi0iUz20A+lPVhPHtNUA= github.com/coder/websocket v1.8.15/go.mod h1:NX3SzP+inril6yawo5CQXx8+fk145lPDC6pumgx0mVg= -github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/containerd/errdefs v1.0.0 h1:tg5yIfIlQIrxYtu9ajqY42W3lpS19XqdxRQeEwYG8PI= +github.com/containerd/errdefs v1.0.0/go.mod h1:+YBYIdtsnF4Iw6nWZhJcqGSg/dwvV7tyJ/kCkyJ2k+M= +github.com/containerd/errdefs/pkg v0.3.0 h1:9IKJ06FvyNlexW690DXuQNx2KA2cUJXx151Xdx3ZPPE= +github.com/containerd/errdefs/pkg v0.3.0/go.mod h1:NJw6s9HwNuRhnjJhM7pylWwMyAkmCQvQ4GpJHEqRLVk= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dennwc/iters v1.2.2 h1:XH2/Etihiy9ZvPOVCR+icQXeYlhbvS7k0qro4x/2qQo= +github.com/dennwc/iters v1.2.2/go.mod h1:M9KuuMBeyEXYTmB7EnI9SCyALFCmPWOIxn5W1L0CjGg= +github.com/distribution/reference v0.6.0 h1:0IXCQ5g4/QMHHkarYzh5l+u8T3t73zM5QvfrDyIgxBk= +github.com/distribution/reference v0.6.0/go.mod h1:BbU0aIcezP1/5jX/8MP0YiH4SdvB5Y4f/wlDRiLyi3E= +github.com/docker/go-connections v0.7.0 h1:6SsRfJddP22WMrCkj19x9WKjEDTB+ahsdiGYf0mN39c= +github.com/docker/go-connections v0.7.0/go.mod h1:no1qkHdjq7kLMGUXYAduOhYPSJxxvgWBh7ogVvptn3Q= +github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= +github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/elliotchance/orderedmap/v3 v3.1.1 h1:eV7lfZ5fVL8d36b8Wogqi/eqm7R/kZcftA9Yiyj+63M= +github.com/elliotchance/orderedmap/v3 v3.1.1/go.mod h1:G+Hc2RwaZvJMcS4JpGCOyViCnGeKf0bTYCGTO4uhjSo= +github.com/fatih/color v1.19.0 h1:Zp3PiM21/9Ld6FzSKyL5c/BULoe/ONr9KlbYVOfG8+w= +github.com/fatih/color v1.19.0/go.mod h1:zNk67I0ZUT1bEGsSGyCZYZNrHuTkJJB+r6Q9VuMi0LE= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= +github.com/florianl/go-tc v0.4.8 h1:hgmakUX1Nm0Ba1I0ZkbUl9CH6HbRwqSiwipnpmYp3Es= +github.com/florianl/go-tc v0.4.8/go.mod h1:B8GeOEnmrbOnxZtaCvsYJcgIzzmM8c/AIhtfCZsDj3Q= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= +github.com/frostbyte73/core v0.1.1 h1:ChhJOR7bAKOCPbA+lqDLE2cGKlCG5JXsDvvQr4YaJIA= +github.com/frostbyte73/core v0.1.1/go.mod h1:mhfOtR+xWAvwXiwor7jnqPMnu4fxbv1F2MwZ0BEpzZo= +github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= +github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= +github.com/gammazero/deque v1.2.1 h1:9fnQVFCCZ9/NOc7ccTNqzoKd1tCWOqeI05/lPqFPMGQ= +github.com/gammazero/deque v1.2.1/go.mod h1:5nSFkzVm+afG9+gy0VIowlqVAW4N8zNcMne+CMQVD2g= +github.com/gammazero/workerpool v1.2.1 h1:MEDvUJsNYGuCvl1RwIXNKu2YtQtHqCSF9XWF04N7lqs= +github.com/gammazero/workerpool v1.2.1/go.mod h1:E32GVRUanF4d6QtRmdss3AScgaDkIyrvPtgRQUWgmx4= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= +github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= +github.com/golang-jwt/jwt/v5 v5.3.1 h1:kYf81DTWFe7t+1VvL7eS+jKFVWaUnK9cB1qbwn63YCY= +github.com/golang-jwt/jwt/v5 v5.3.1/go.mod h1:fxCRLWMO43lRc8nhHWY6LGqRcf+1gQWArsqaEUEa5bE= +github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= +github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= +github.com/google/cel-go v0.30.0 h1:ll54AkzKunWkBn9wSoiUXbFZXYZTkdJGNXTBXUoolGo= +github.com/google/cel-go v0.30.0/go.mod h1:X0bD6iVNR8pkROSOoHVdgTkzmRcosof7WQqCD6wcMc8= +github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.7.0 h1:wk8382ETsv4JYUZwIsn6YpYiWiBsYLSJiTsyBybVuN8= +github.com/google/go-cmp v0.7.0/go.mod h1:pXiqmnSA92OHEEa9HXL2W4E7lf9JzCmGVUdgjX3N/iU= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo= github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB11/k= +github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= +github.com/hashicorp/go-retryablehttp v0.7.8 h1:ylXZWnqa7Lhqpk0L1P1LzDtGcCR0rPVUrx/c8Unxc48= +github.com/hashicorp/go-retryablehttp v0.7.8/go.mod h1:rjiScheydd+CxvumBsIrFKlx3iS0jrZ7LvzFGFmuKbw= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jellydator/ttlcache/v3 v3.4.1 h1:bOdXmXiycyK6E6Qjyuj5vl+/vU3SCOoDs8a86NbHjAQ= +github.com/jellydator/ttlcache/v3 v3.4.1/go.mod h1:j7LO12PNghFg5+0v9budMAT4rDK4JY969jb9vOdOBBk= +github.com/josharian/native v0.0.0-20200817173448-b6b71def0850/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= +github.com/josharian/native v1.0.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= +github.com/josharian/native v1.1.0 h1:uuaP0hAbW7Y4l0ZRQ6C9zfb7Mg1mbFKry/xzDAfmtLA= +github.com/josharian/native v1.1.0/go.mod h1:7X/raswPFr05uY3HiLlYeyQntB6OO7E/d2Cu7qoaN2w= +github.com/jsimonetti/rtnetlink v0.0.0-20190606172950-9527aa82566a/go.mod h1:Oz+70psSo5OFh8DBl0Zv2ACw7Esh6pPUphlvZG9x7uw= +github.com/jsimonetti/rtnetlink v0.0.0-20200117123717-f846d4f6c1f4/go.mod h1:WGuG/smIU4J/54PblvSbh+xvCZmpJnFgr3ds6Z55XMQ= +github.com/jsimonetti/rtnetlink v0.0.0-20201009170750-9c6f07d100c1/go.mod h1:hqoO/u39cqLeBLebZ8fWdE96O7FxrAsRYhnVOdgHxok= +github.com/jsimonetti/rtnetlink v0.0.0-20201216134343-bde56ed16391/go.mod h1:cR77jAZG3Y3bsb8hF6fHJbFoyFukLFOkQ98S0pQz3xw= +github.com/jsimonetti/rtnetlink v0.0.0-20201220180245-69540ac93943/go.mod h1:z4c53zj6Eex712ROyh8WI0ihysb5j2ROyV42iNogmAs= +github.com/jsimonetti/rtnetlink v0.0.0-20210122163228-8d122574c736/go.mod h1:ZXpIyOK59ZnN7J0BV99cZUPmsqDRZ3eq5X+st7u/oSA= +github.com/jsimonetti/rtnetlink v0.0.0-20210212075122-66c871082f2b/go.mod h1:8w9Rh8m+aHZIG69YPGGem1i5VzoyRC8nw2kA8B+ik5U= +github.com/jsimonetti/rtnetlink v0.0.0-20210525051524-4cc836578190/go.mod h1:NmKSdU4VGSiv1bMsdqNALI4RSvvjtz65tTMCnD05qLo= +github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786 h1:N527AHMa793TP5z5GNAn/VLPzlc0ewzWdeP/25gDfgQ= +github.com/jsimonetti/rtnetlink v0.0.0-20211022192332-93da33804786/go.mod h1:v4hqbTdfQngbVSZJVWUhGE/lbTFf9jb+ygmNUDQMuOs= +github.com/jxskiss/base62 v1.1.0 h1:A5zbF8v8WXx2xixnAKD2w+abC+sIzYJX+nxmhA6HWFw= +github.com/jxskiss/base62 v1.1.0/go.mod h1:HhWAlUXvxKThfOlZbcuFzsqwtF5TcqS9ru3y5GfjWAc= +github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJnhSlk= +github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= +github.com/klauspost/cpuid/v2 v2.4.0 h1:S6Hrbc7+ywsr0r+RLapfGBHfyefhCTwEh3A0tV913Dw= +github.com/klauspost/cpuid/v2 v2.4.0/go.mod h1:19jmZ9mjzoF//ddRSUsv0zfBTJWh3QJh9FNxZTMrGxU= github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= github.com/koron/go-ssdp v0.0.4/go.mod h1:oDXq+E5IL5q0U8uSBcoAXzTzInwy5lEgC91HoKtbmZk= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/libp2p/go-netroute v0.2.1 h1:V8kVrpD8GK0Riv15/7VN6RbUQ3URNZVosw7H2v9tksU= github.com/libp2p/go-netroute v0.2.1/go.mod h1:hraioZr0fhBjG0ZRXJJ6Zj2IVEVNx6tDTFQfSmcq7mQ= +github.com/lithammer/shortuuid/v4 v4.2.0 h1:LMFOzVB3996a7b8aBuEXxqOBflbfPQAiVzkIcHO0h8c= +github.com/lithammer/shortuuid/v4 v4.2.0/go.mod h1:D5noHZ2oFw/YaKCfGy0YxyE7M0wMbezmMjPdhyEFe6Y= +github.com/livekit/livekit-server v1.13.6 h1:vvLufsmzMG9ueZ6BqSahasC8rUH1toSv4979fOY9uQI= +github.com/livekit/livekit-server v1.13.6/go.mod h1:Dxd+bHclL+F08/T8TlZkxjSRYXCHObLo+ujPRc7h7MM= +github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731 h1:9x+U2HGLrSw5ATTo469PQPkqzdoU7be46ryiCDO3boc= +github.com/livekit/mageutil v0.0.0-20250511045019-0f1ff63f7731/go.mod h1:Rs3MhFwutWhGwmY1VQsygw28z5bWcnEYmS1OG9OxjOQ= +github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095 h1:BcliKAXoMhl/nWmzQweQ5kmh4Qqagxl4s3Z5pvM/7AY= +github.com/livekit/mediatransportutil v0.0.0-20260821083140-f234b534b095/go.mod h1:o8CFmAdrVwzJNOCsQCLUzXRjokkufNshnQHOe4fRaqU= +github.com/livekit/protocol v1.50.5-0.20260821144838-17c16cf496fd h1:pGWIR9fw1eC699v2njRX/7Y+baEXB7p6k9CvDg2uVrQ= +github.com/livekit/protocol v1.50.5-0.20260821144838-17c16cf496fd/go.mod h1:h4wP4IBPkT5aew7SZSGViHSmanKAiKWsHij/5YfL4BE= +github.com/livekit/psrpc v0.7.3 h1:bekuZt/ZQzg8+/M8G6G5jq7bvV9fAKdPHSOZeTwrIIc= +github.com/livekit/psrpc v0.7.3/go.mod h1:rAI+m2+/cb4x9RXhLRtUx5ZwdfjjXOl4zi46IjEetaw= +github.com/livekit/server-sdk-go/v2 v2.14.0 h1:yy7LsR1+64DHC2i4DxZe1xX4ScVq+S3CvgpvHRruEec= +github.com/livekit/server-sdk-go/v2 v2.14.0/go.mod h1:+HCKTpzV21b/jvBtu+OmWbquUxaL74kHLI9ZwKmdhKU= github.com/lucasb-eyer/go-colorful v1.4.1 h1:1EO+WB73+EH8EVbzlrG3KLAfEypQWVHIBqlTf+2hNss= github.com/lucasb-eyer/go-colorful v1.4.1/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0= +github.com/mackerelio/go-osstat v0.2.8 h1:I2duicTaCGWoM53XwAwA9OIe1inu0xnVs8/pqOWWVr4= +github.com/mackerelio/go-osstat v0.2.8/go.mod h1:SyS3XxKdoSKJnTGTkN5Yrh6VUQVuAURACfE6y+2DN4k= +github.com/mattn/go-colorable v0.1.15 h1:+u9SLTRGnXv73cEsnsmoZBom+dMU88B2M0aDcWy0/jY= +github.com/mattn/go-colorable v0.1.15/go.mod h1:6LmQG8QLFO4G5z1gPvYEzlUgJ2wF+stgPZH1UqBm1s8= github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI= github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A= github.com/mattn/go-runewidth v0.0.24 h1:cpokDiIn0MGnhdHwuWnJBITySJ20QyNGnY2kR/ay2DU= github.com/mattn/go-runewidth v0.0.24/go.mod h1:XBkDxAl56ILZc9knddidhrOlY5R/pDhgLpndooCuJAs= +github.com/mdlayher/ethtool v0.0.0-20210210192532-2b88debcdd43/go.mod h1:+t7E0lkKfbBsebllff1xdTmyJt8lH37niI6kwFk9OTo= +github.com/mdlayher/genetlink v1.0.0/go.mod h1:0rJ0h4itni50A86M2kHcgS85ttZazNt7a8H2a2cw0Gc= +github.com/mdlayher/netlink v0.0.0-20190409211403-11939a169225/go.mod h1:eQB3mZE4aiYnlUsyGGCOpPETfdQq4Jhsgf1fk3cwQaA= +github.com/mdlayher/netlink v1.0.0/go.mod h1:KxeJAFOFLG6AjpyDkQ/iIhxygIUKD+vcwqcnu43w/+M= +github.com/mdlayher/netlink v1.1.0/go.mod h1:H4WCitaheIsdF9yOYu8CFmCgQthAPIWZmcKp9uZHgmY= +github.com/mdlayher/netlink v1.1.1/go.mod h1:WTYpFb/WTvlRJAyKhZL5/uy69TDDpHHu2VZmb2XgV7o= +github.com/mdlayher/netlink v1.2.0/go.mod h1:kwVW1io0AZy9A1E2YYgaD4Cj+C+GPkU6klXCMzIJ9p8= +github.com/mdlayher/netlink v1.2.1/go.mod h1:bacnNlfhqHqqLo4WsYeXSqfyXkInQ9JneWI68v1KwSU= +github.com/mdlayher/netlink v1.2.2-0.20210123213345-5cc92139ae3e/go.mod h1:bacnNlfhqHqqLo4WsYeXSqfyXkInQ9JneWI68v1KwSU= +github.com/mdlayher/netlink v1.3.0/go.mod h1:xK/BssKuwcRXHrtN04UBkwQ6dY9VviGGuriDdoPSWys= +github.com/mdlayher/netlink v1.4.0/go.mod h1:dRJi5IABcZpBD2A3D0Mv/AiX8I9uDEu5oGkAVrekmf8= +github.com/mdlayher/netlink v1.4.1/go.mod h1:e4/KuJ+s8UhfUpO9z00/fDZZmhSrs+oxyqAS9cNgn6Q= +github.com/mdlayher/netlink v1.6.2/go.mod h1:O1HXX2sIWSMJ3Qn1BYZk1yZM+7iMki/uYGGiwGyq/iU= +github.com/mdlayher/netlink v1.11.2 h1:HKh2jqe+omdSWcQ88nrT7INE61B0NXfiSPFdgL4YbNI= +github.com/mdlayher/netlink v1.11.2/go.mod h1:uT2Yc/QLaZubzDpZIBi9d4GoeLwtp3x1AMeqSRrK2sA= +github.com/mdlayher/socket v0.0.0-20210307095302-262dc9984e00/go.mod h1:GAFlyu4/XV68LkQKYzKhIo/WW7j3Zi0YRAz/BOoanUc= +github.com/mdlayher/socket v0.2.3/go.mod h1:bz12/FozYNH/VbvC3q7TRIK/Y6dH1kCKsXaUeXi/FmY= +github.com/mdlayher/socket v0.6.1 h1:M7uj2NtuujUY4mYr1C57NmfNiRHbkKpnBxO856lsc3A= +github.com/mdlayher/socket v0.6.1/go.mod h1:+/SGtqc9V+5dAuRgQsU0fGBI+oRDiW7O2Obx10OIWfg= +github.com/moby/docker-image-spec v1.3.1 h1:jMKff3w6PgbfSa69GfNg+zN/XLhfXJGnEx3Nl2EsFP0= +github.com/moby/docker-image-spec v1.3.1/go.mod h1:eKmb5VW8vQEh/BAr2yvVNvuiJuY6UIocYsFu/DxxRpo= +github.com/moby/moby/api v1.55.0 h1:2/sexvQyqIWS8pRSCFddBfpW2qE7vR7FCL+vN8pxwMc= +github.com/moby/moby/api v1.55.0/go.mod h1:+RQ6wluLwtYaTd1WnPLykIDPekkuyD/ROWQClE83pzs= +github.com/moby/moby/client v0.5.1 h1:tYNaJno4c0HXz12y5BiqEDy0rVTYkWzI26lGvnTMiJw= +github.com/moby/moby/client v0.5.1/go.mod h1:odLstlZ6uSnfvAgVxMpvgmb8SUdd+siH2T0GBuxVAlM= github.com/muesli/cancelreader v0.2.2 h1:3I4Kt4BQjOR54NavqnDogx/MIoWBFa0StPA8ELUXHmA= github.com/muesli/cancelreader v0.2.2/go.mod h1:3XuTXfFS2VjM+HTLZY9Ak0l6eUKfijIfMUZ4EgX0QYo= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= +github.com/nats-io/nats.go v1.52.0 h1:n3avV4VBsCgsdwh71TppsTwtv+QdPs7ntSKM8qJLGsc= +github.com/nats-io/nats.go v1.52.0/go.mod h1:26HypzazeOkyO3/mqd1zZd53STJN0EjCYF9Uy2ZOBno= +github.com/nats-io/nkeys v0.4.16 h1:rd5oAuLOb8mnAycB0xleuEBNS1pVVnN0fv/FF34Eypg= +github.com/nats-io/nkeys v0.4.16/go.mod h1:llLgWoI0o4z/Q57q2R1kHfmocyhGV6VG/U18Glg1Afs= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w= github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls= github.com/netbirdio/go-nat v0.0.0-20260821095157-6b2c8c5c74e8 h1:pBxXEsxcsO3qVUND//5j1kelYlO57x5IrRviNF0+0iA= github.com/netbirdio/go-nat v0.0.0-20260821095157-6b2c8c5c74e8/go.mod h1:mFViabv4PpnoDw9w7W21a7xux6APA4q7KQZRsv4BCl8= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/image-spec v1.1.1 h1:y0fUlFfIZhPF1W537XOLg0/fcx6zcHCJwooC2xJA040= +github.com/opencontainers/image-spec v1.1.1/go.mod h1:qpqAh3Dmcf36wStyyWU+kCeDgrGnAve2nCC8+7h8Q0M= +github.com/ory/dockertest/v4 v4.0.0 h1:i19aFsO/VXE0VrMk4ifnKW4G/KIJ93PCjLOslxXoPME= +github.com/ory/dockertest/v4 v4.0.0/go.mod h1:b5Ofu8VIxWNhXFvQcLu17pRNQdoUBKtXBW74G4Ygzx8= +github.com/petermattis/goid v0.0.0-20260725062400-500c67a39b75 h1:VmZ6mKVkxavKEhEy4ZYyV7BwBYBFBP0TwIqmLk84fpU= +github.com/petermattis/goid v0.0.0-20260725062400-500c67a39b75/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/pion/datachannel v1.6.2 h1:7EXQ8TH3vTouBUdRWYbcX2edSx9Yj6k5zl5P+qyxEPc= github.com/pion/datachannel v1.6.2/go.mod h1:pzbdAZvyGtXbcHM1hBbsFaOTf40lZizU/dNlvVOak6E= github.com/pion/dtls/v3 v3.1.5 h1:9xJtVsHwMYeSjPp5Hh1FTis4DchnQWtnOa5o+6ygqfc= @@ -78,63 +259,175 @@ github.com/pion/sctp v1.11.1 h1:O4dIFyURw1KTST7w+gtD4gLeYXkhPa0xXLHMMoe/OSA= github.com/pion/sctp v1.11.1/go.mod h1:7KFmTwLcoYgJs/Z+99nJvsWL0qDpuyloSI0RbAqlrz0= github.com/pion/sdp/v3 v3.0.19 h1:1VMKs3gIkTQV5M3hNKfTAPrDXSNrYtOlmOD8+mSZUGQ= github.com/pion/sdp/v3 v3.0.19/go.mod h1:dE5WOSlzXrtiE/iuZqe9n+AcEbOjtAd3k5m5NtlV/qU= -github.com/pion/srtp/v3 v3.0.12 h1:U7V17bckl7sI4mb3sepiojByDuBY0wNCqQE+6IlQBbc= -github.com/pion/srtp/v3 v3.0.12/go.mod h1:EeZOi/sd6glM1EXapg051gdNWO9yWT1YSsgQ4SlJkns= +github.com/pion/srtp/v3 v3.0.13 h1:FmQaqgNbN1vUtMhEsmj8trldc3lNZr1xmN7nl8CyX+Q= +github.com/pion/srtp/v3 v3.0.13/go.mod h1:7qR3L69t8RX0EPVQwGNwCa1Gy9keKKNDpWwQzZbeXDY= github.com/pion/stun/v3 v3.1.6 h1:WnhsD0eHCiwCfKNkVx0VJJwr2Y3eV4Ueih3KJ+dfZy8= github.com/pion/stun/v3 v3.1.6/go.mod h1:zRUghXSQU32Lx5orJsz3uYMkIihweXb3mu5gIns02fs= github.com/pion/transport/v3 v3.1.1 h1:Tr684+fnnKlhPceU+ICdrw6KKkTms+5qHMgw6bIkYOM= github.com/pion/transport/v3 v3.1.1/go.mod h1:+c2eewC5WJQHiAA46fkMMzoYZSuGzA/7E2FPrOYHctQ= -github.com/pion/transport/v4 v4.0.2 h1:ifYlPqNwsy6aKQ9y8yzxXlHae5431ZrH2avkD/Rn6Tk= -github.com/pion/transport/v4 v4.0.2/go.mod h1:06hFI+jCFcok2X2MekVufNZ/uzNZXivGBPfviSVcjgM= +github.com/pion/transport/v4 v4.1.0 h1:8S+nF2reM2cJuqC6g78OVy2BBgmbdns+acx3jA97BvQ= +github.com/pion/transport/v4 v4.1.0/go.mod h1:06hFI+jCFcok2X2MekVufNZ/uzNZXivGBPfviSVcjgM= github.com/pion/turn/v5 v5.0.12 h1:6+b69ivQQXSlyfkp2AKripqD2k3W32qXK8QzCzpJWPI= github.com/pion/turn/v5 v5.0.12/go.mod h1:CQACsRDJtjQ+6RSrGHrS2PCIerLwbW3uqXRqOvtjAFg= github.com/pion/webrtc/v4 v4.2.18 h1:smA/3g6Gy4RohM0VIZ5KKY/12TQbxv3XFgpUMyb2EUI= github.com/pion/webrtc/v4 v4.2.18/go.mod h1:vmzi6s+rvhoIuT94DPqivB+0xJXs9rG4QRD+4MgBtlY= -github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= -github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= +github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/prometheus/client_golang v1.24.1 h1:JnJkREXzWxUdCuPFpIWZiPispT9xVV59uiuyR2bPlnU= +github.com/prometheus/client_golang v1.24.1/go.mod h1:F+oSRECHg4sse5ucfYpYDeIv/hu68Zo0uoHKetWnzcE= +github.com/prometheus/client_model v0.6.2 h1:oBsgwpGs7iVziMvrGhE53c/GrLUsZdHnqNwqPLxwZyk= +github.com/prometheus/client_model v0.6.2/go.mod h1:y3m2F6Gdpfy6Ut/GBsUqTWZqCUvMVzSfMLjcu6wAwpE= +github.com/prometheus/common v0.70.1 h1:1HvjP4D5oL3t8RsPlwxA9onvvStjtIHYE5XuuwOi/PY= +github.com/prometheus/common v0.70.1/go.mod h1:VdFUQDMZK3VLkurFUVhia6uys/0suUp86TJz5qbJRhc= +github.com/prometheus/procfs v0.21.1 h1:GljZCt+zSTS+NZq88cyQ1LjZ+RCHp3uVuabBWA5+OJI= +github.com/prometheus/procfs v0.21.1/go.mod h1:aB55Cww9pdSJVHk0hUf0inxWyyjPogFIjmHKYgMKmtY= +github.com/puzpuzpuz/xsync/v4 v4.5.0 h1:vOSWu6b57/emh+L/Cw0BeQfvxa/cogFywXHeGUxQxAg= +github.com/puzpuzpuz/xsync/v4 v4.5.0/go.mod h1:VJDmTCJMBt8igNxnkQd86r+8KUeN1quSfNKu5bLYFQo= +github.com/redis/go-redis/v9 v9.22.0 h1:laDvpYXTJtZLloinw1fA5Kqd6HAEH2XKxOkG/PDq2F0= +github.com/redis/go-redis/v9 v9.22.0/go.mod h1:y2g0Wj8rQvuK0ELM+oxSudcLtC09JScs98I/X9gRWY4= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE= github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ= github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= -github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U= +github.com/rodaine/protogofakeit v0.1.1 h1:ZKouljuRM3A+TArppfBqnH8tGZHOwM/pjvtXe9DaXH8= +github.com/rodaine/protogofakeit v0.1.1/go.mod h1:pXn/AstBYMaSfc1/RqH3N82pBuxtWgejz1AlYpY1mI0= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rogpeppe/go-internal v1.15.0 h1:D0RCU5rMAp+SpgkiNdrjfJ+LX4J1M32V2NeCY7EJ6hc= +github.com/rogpeppe/go-internal v1.15.0/go.mod h1:DrUVZyrJU+txYW5/1kwtXQSMFio52ZOxX7yM1VHvnxs= +github.com/shoenig/test v1.7.0 h1:eWcHtTXa6QLnBvm0jgEabMRN/uJ4DMV3M8xUGgRkZmk= +github.com/shoenig/test v1.7.0/go.mod h1:UxJ6u/x2v/TNs/LoLxBNJRV9DiwBBKYxXSyczsBHFoI= +github.com/stretchr/testify v1.12.0 h1:K6Mr6jO9JICuend/5xzTM03ydSV3vdNRYAdPSukj8uI= +github.com/stretchr/testify v1.12.0/go.mod h1:bOYBZb5qJ00vPzWfIqBUZPaxK8jWiXc6d3ErP4Ca9Gw= github.com/thesyncim/gopus v0.1.1 h1:SM58S7VZGnEZ3ZR0pX5AoC33//LBxJhKkvm15MbBmik= github.com/thesyncim/gopus v0.1.1/go.mod h1:orRqwrGs5gqYRRnhqwI0Y3liqQTeDkreUpra+Kv9bQc= +github.com/twitchtv/twirp v8.1.3+incompatible h1:+F4TdErPgSUbMZMwp13Q/KgDVuI7HJXP61mNV3/7iuU= +github.com/twitchtv/twirp v8.1.3+incompatible/go.mod h1:RRJoFSAmTEh2weEqWtpPE3vFK5YBhA6bqp2l1kfCC5A= github.com/wlynxg/anet v0.0.5 h1:J3VJGi1gvo0JwZ/P1/Yc/8p63SoW98B5dHkYDmpgvvU= github.com/wlynxg/anet v0.0.5/go.mod h1:eay5PRQr7fIVAMbTbchTnO9gG65Hg/uYGdc7mguHxoA= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e h1:JVG44RsyaB9T2KIHavMF/ppJZNG9ZpyihvCd0w101no= github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e/go.mod h1:RbqR21r5mrJuqunuUZ/Dhy/avygyECGrLceyNeo4LiM= +github.com/zeebo/assert v1.3.0 h1:g7C04CbJuIDKNPFHmsk4hwZDO5O+kntRxzaUoNXj+IQ= +github.com/zeebo/assert v1.3.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/xxh3 v1.1.0 h1:s7DLGDK45Dyfg7++yxI0khrfwq9661w9EN78eP/UZVs= +github.com/zeebo/xxh3 v1.1.0/go.mod h1:IisAie1LELR4xhVinxWS5+zf1lA4p0MW4T+w+W07F5s= +go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= +go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0 h1:8tvICD4vSTOOsNrsI4Ljf6C+6UKvpTEH5XY3JMoyPoo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.69.0/go.mod h1:z9+yiacE0IHRqM4qFfkbt/JYlmYXgss8GY/jXoNuPJI= +go.opentelemetry.io/otel v1.45.0 h1:pdrWmLHofpubmArBv1LgFSv1Z0Ie/ppdZzu+kUN5EeU= +go.opentelemetry.io/otel v1.45.0/go.mod h1:XZxIqPapzEYnhNSScF5DIqXhm/rYi0FzCe2XddAwZfQ= +go.opentelemetry.io/otel/metric v1.45.0 h1:7Eg1uH7CJ5cXv9is6tnBe1FI6rj1nwUdbFypRm3br/M= +go.opentelemetry.io/otel/metric v1.45.0/go.mod h1:HAPbm1nd3p1PmFH7v2dR+6BjXxw+Lq4a2+pndMAm08s= +go.opentelemetry.io/otel/sdk v1.44.0 h1:nHYwb9lK+fJPU/dnT6s7W7Z8itMWyqrnVfbheVYrZ58= +go.opentelemetry.io/otel/sdk v1.44.0/go.mod h1:Osuydd3Se74nqjAKxid74N5eC+jfEqfTegHRnq58oK0= +go.opentelemetry.io/otel/sdk/metric v1.44.0 h1:3LlKgI+VjbVsjNRFZJZAJ30WjXC5VkNRks6si09iEfI= +go.opentelemetry.io/otel/sdk/metric v1.44.0/go.mod h1:5B5pMARnXxKhltooO4xUuCBorl65a4EpnTalObqOigA= +go.opentelemetry.io/otel/trace v1.45.0 h1:l/mP6Uv7oNO7/TblbhpbgMidxhq1uO/rPsikOyVhxag= +go.opentelemetry.io/otel/trace v1.45.0/go.mod h1:qoJJA2xNMnxRrdISU/kLtfUH2wNeQbiv+jhs/CxI8bc= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= +go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.28.0 h1:IZzaP1Fv73/T/pBMLk4VutPl36uNC+OSUh3JLG3FIjo= +go.uber.org/zap v1.28.0/go.mod h1:rDLpOi171uODNm/mxFcuYWxDsqWSAVkFdX4XojSKg/Q= +go.uber.org/zap/exp v0.3.0 h1:6JYzdifzYkGmTdRR59oYH+Ng7k49H9qVpWwNSsGJj3U= +go.uber.org/zap/exp v0.3.0/go.mod h1:5I384qq7XGxYyByIhHm6jg5CHkGY0nsTfbDLgDDlgJQ= +go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= +go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= +go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= +go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts= -golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.54.0 h1:YLIA59K4fiNzHzjnZt2tUJQjQtUWfWbeHBqKtk3eScw= +golang.org/x/crypto v0.54.0/go.mod h1:KWL8ny2AZdGR2cWmzeHrp2azQPGogOv+HeQaVEXC2dk= +golang.org/x/exp v0.0.0-20260727155853-b88d891fe743 h1:ex206bKw+v3K0dm3andkrIF+ijyQKJG1pLgwQ2PYdQM= +golang.org/x/exp v0.0.0-20260727155853-b88d891fe743/go.mod h1:EdfpwwqSu+0Li0mzskwHU6FWDV3t9Q+RZDo3QMUtL3Q= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.38.0 h1:MECBjubtXD7yj4HrhIUcywNaGeNVUdfVnxmPajOk4yk= golang.org/x/mod v0.38.0/go.mod h1:V6Xz0pq8TQ3dGqVQ1FVHuelZpAL0uNhSkk9ogYP3c40= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.50.0 h1:ucWh9eiCGyDR3vtzso0WMQinm2Dnt8cFMuQa9K33J60= -golang.org/x/net v0.50.0/go.mod h1:UgoSli3F/pBgdJBHCTc+tp3gmrU4XswgGRgtnwWTfyM= +golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20200202094626-16171245cfb2/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20201010224723-4f7140c49acb/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201216054612-986b41b23924/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220923203811-8be639271d50/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.57.0 h1:K5+3DljvIuDG9/Jv9rvyMywYNFCQ9RSUY6OOTTkT+tE= +golang.org/x/net v0.57.0/go.mod h1:KpXc8iv+r3XplLAG/f7Jsf9RPszJzdR0f58q9vGOuEU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220923202941-7f9b1623fab7/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.22.0 h1:SZjpbeLmrCk4xhRSZFNZW5gFUeCeFgjekvI/+gfScek= golang.org/x/sync v0.22.0/go.mod h1:9xrNwdLfx4jkKbNva9FpL6vEN7evnE43NNNJQ2LF3+0= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190411185658-b44545bcd369/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201118182958-a01c418693c7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210110051926-789bb1bd4061/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210123111255-9b0068b26619/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210216163648-f7da38b97c65/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210525143221-35b2ab0089ea/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.47.0 h1:o7XGOvZQCADBQQ4Y7VNq2dRWQR7JmOUW8Kxx4ZsNgWs= golang.org/x/sys v0.47.0/go.mod h1:4GL1E5IUh+htKOUEOaiffhrAeqysfVGipDYzABqnCmw= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.41.0 h1:vz/seA0lnX87Othu2f/0L24RcgrXD9/YFTSuGjj3rH8= golang.org/x/text v0.41.0/go.mod h1:jvf1O8ajNzZqhSrQBPbutR/EB83Cc0CFrezNQIwbb5M= -golang.org/x/time v0.14.0 h1:MRx4UaLrDotUKUdCIqzPC48t1Y9hANFKIRpNx+Te8PI= -golang.org/x/time v0.14.0/go.mod h1:eL/Oa2bBBK0TkX57Fyni+NgnyQQN4LitPmob2Hjnqw4= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.48.0 h1:3+hClM1aLL5mjMKm5ovokw9epgRXPuu2tILgismM6RE= golang.org/x/tools v0.48.0/go.mod h1:08xX0orndb/F7jJxGDicx061tyd5pcMto75YMAXr6lk= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4= +gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E= +google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d h1:FarXi840EJWSHYTN3ERkADbPWjl307+FGrA22KAVjjc= +google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d/go.mod h1:K/+WGbmBY7aNW1HDw1fJnKYo10i0DkAX6pows00dLig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d h1:IL4hdHzcUv2l/gcg98/Rj3FbtE6axwqslOW8SW0C+S0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8= +google.golang.org/grpc v1.83.0 h1:JeNZEKJFbQxArAMl+hiytHauacDNqJUllNfmIMmpqnQ= +google.golang.org/grpc v1.83.0/go.mod h1:kDyl6SKsiHKt0uylY5gtn5cEjkrIOhQOGDgIc4JGwzQ= +google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE= +google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= modernc.org/cc/v4 v4.29.2 h1:h6+9ciCnPKutf4I03CvheAvDLX7+IHlqR6Iy6J+cgd8= diff --git a/internal/client/clientapp/app.go b/internal/client/clientapp/app.go index 4b17505..85d7648 100644 --- a/internal/client/clientapp/app.go +++ b/internal/client/clientapp/app.go @@ -5,12 +5,15 @@ import ( "encoding/json" "errors" "fmt" + "log" "log/slog" "net" "net/http" "net/url" + "os" "path/filepath" "strings" + "syscall" "time" "github.com/TNTcraftHIM/Screener/internal/client/browser" @@ -21,6 +24,7 @@ import ( "github.com/TNTcraftHIM/Screener/internal/client/nativecapture" "github.com/TNTcraftHIM/Screener/internal/client/nativecontrol" "github.com/TNTcraftHIM/Screener/internal/client/publictunnel" + "github.com/TNTcraftHIM/Screener/internal/diagnostics" serverapp "github.com/TNTcraftHIM/Screener/internal/server/app" serverconfig "github.com/TNTcraftHIM/Screener/internal/server/config" "github.com/TNTcraftHIM/Screener/internal/server/webassets" @@ -48,23 +52,62 @@ type Options struct { LANAddress string Port int DisableBrowser bool + Debug bool + LogDir string CaptureProcess string TunnelProcess string Ready func(string) console *clientConsole + logger *slog.Logger } func Run(ctx context.Context, options Options) (returnedErr error) { ctx, cancel := context.WithCancel(ctx) + options.Debug = options.Debug || clientDebugEnabled(os.Getenv("SCREENER_DEBUG")) options.console = newClientConsole(cancel, options.DisableBrowser) + var recorder *diagnostics.Recorder + var restoreLogger func() defer func() { cancel() + if recorder != nil { + slog.Debug("screener-client", "event", "stopped", "failed", returnedErr != nil) + if options.console.program == nil { + path, err := recorder.Export() + options.console.send(consoleExportResult{path: path, err: err}) + returnedErr = errors.Join(returnedErr, err) + } + restoreLogger() + returnedErr = errors.Join(returnedErr, recorder.Close()) + } returnedErr = errors.Join(returnedErr, options.console.finish(returnedErr)) }() + if options.Debug { + var err error + recorder, err = openClientDiagnostics(options.LogDir) + if err != nil { + return fmt.Errorf("Screener Client diagnostics are unavailable: %w", err) + } + previous, previousWriter, previousFlags := slog.Default(), log.Writer(), log.Flags() + restoreLogger = func() { + slog.SetDefault(previous) + log.SetOutput(previousWriter) + log.SetFlags(previousFlags) + } + options.logger = recorder.Logger() + slog.SetDefault(options.logger) + // Only structured application events belong in the persisted log. + log.SetOutput(previousWriter) + log.SetFlags(previousFlags) + options.console.send(consoleDebug{logPath: recorder.LogPath(), export: recorder.Export}) + } + slog.Debug("screener-client", "event", "start", "revision", BuildRevision) options.console.show(consoleView{state: "starting"}) if err := validateMode(options); err != nil { return err } + if options.Port == 0 { + options.Port = DefaultLocalPort + } configPath := strings.TrimSpace(options.ConfigPath) if configPath == "" { var err error @@ -81,12 +124,17 @@ func Run(ctx context.Context, options Options) (returnedErr error) { if err != nil { return err } + slog.Debug("screener-client", "event", "configuration", "siteConfigured", config.Site != "", "localAccessProtected", config.LocalAccessPassword != "") if options.SiteSet || options.Local { if err = clientconfig.Save(configPath, config); err != nil { return errors.New("Screener Client configuration is unavailable") } } nativeMedia := discoverNativeMedia(ctx, options.CaptureProcess) + slog.Debug("screener-client", "event", "native-capabilities", + "video", nativeMedia.capabilities.Video, "processAudio", nativeMedia.capabilities.ProcessAudio, + "systemAudio", nativeMedia.capabilities.SystemAudio, "hardwareH264", nativeMedia.capabilities.HardwareH264, + "softwareVP8", nativeMedia.capabilities.SoftwareVP8) client, err := loopback.Start(ctx, loopback.Options{ AllowedOrigins: clientOrigins(config.Site, options.Port), NativeMedia: nativeMedia.capabilities, @@ -97,6 +145,7 @@ func Run(ctx context.Context, options Options) (returnedErr error) { return errors.New("Screener Client could not start") } defer client.Close() + slog.Debug("screener-client", "event", "control-ready") if options.console.machine { if err = printEndpoint(client.Endpoint()); err != nil { return err @@ -247,11 +296,43 @@ func validateMode(options Options) error { return nil } +func clientDebugEnabled(value string) bool { + for _, component := range strings.Split(value, ",") { + if strings.TrimSpace(component) == "client" { + return true + } + } + return false +} + +func openClientDiagnostics(directory string) (*diagnostics.Recorder, error) { + if directory = strings.TrimSpace(directory); directory == "" { + directory = strings.TrimSpace(os.Getenv("SCREENER_LOG_DIR")) + } + if directory != "" { + return diagnostics.Open(directory, "client", BuildRevision) + } + executable, err := os.Executable() + if err != nil { + return nil, err + } + recorder, err := diagnostics.Open(filepath.Join(filepath.Dir(executable), "logs"), "client", BuildRevision) + if err == nil || !errors.Is(err, os.ErrPermission) && !errors.Is(err, syscall.EROFS) { + return recorder, err + } + cache, cacheErr := os.UserCacheDir() + if cacheErr != nil { + return nil, errors.Join(err, cacheErr) + } + return diagnostics.Open(filepath.Join(cache, "Screener", "logs"), "client", BuildRevision) +} + // runSite opens the configured Screener Site in the Browser and waits for the // loopback server, which is the only thing this mode owns. It takes no context: // cancellation reaches it through client.Done(). func runSite(site string, options Options, client *loopback.Server) error { var err error + slog.Debug("screener-client", "event", "mode", "mode", "site") launchURL := clientLaunchURL(site) view := consoleView{mode: "site", state: "starting", entry: launchURL} options.console.show(view) @@ -266,6 +347,7 @@ func runSite(site string, options Options, client *loopback.Server) error { options.Ready(launchURL) } view.state = "ready" + slog.Debug("screener-client", "event", "site-ready") options.console.show(view) if err = <-client.Done(); err != nil { return errors.New("Screener Client stopped unexpectedly") @@ -280,7 +362,17 @@ func runLocal(ctx context.Context, options Options, config clientconfig.Config, if options.Link { view.mode = "link" } + slog.Debug("screener-client", "event", "mode", "mode", view.mode) options.console.show(view) + listener, err := net.ListenTCP("tcp4", &net.TCPAddr{IP: net.IPv4zero, Port: options.Port}) + if err != nil { + return fmt.Errorf("local server port is unavailable: %w", err) + } + defer func() { + if listener != nil { + _ = listener.Close() + } + }() addresses, err := lan.Addresses() if err != nil { return err @@ -302,6 +394,7 @@ func runLocal(ctx context.Context, options Options, config clientconfig.Config, } defer tunnel.Close() publicOrigin = tunnel.Origin() + slog.Debug("screener-client", "event", "public-link-ready") } stunURLs, natPredictionStunURLs := localSTUNURLs(options.Link) @@ -317,27 +410,25 @@ func runLocal(ctx context.Context, options Options, config clientconfig.Config, if err != nil { return err } + logger := options.logger + if logger == nil { + logger = slog.New(slog.NewTextHandler(options.console.logWriter(), nil)) + } localServer, err := serverapp.New(serverapp.Options{ - Config: localConfig, - Assets: webassets.FS(), - // Machine mode inherits stderr; the interactive console shows server - // diagnostics in its log pane, as the Node child's stderr did. - Logger: slog.New(slog.NewTextHandler(options.console.logWriter(), nil)), + Config: localConfig, + Listener: listener, + Assets: webassets.FS(), + Logger: logger, }) if err != nil { return err } + listener = nil // Ownership is now in the application's startup/shutdown path. + defer func() { _ = endLocalServer(localServer) }() if _, err = localServer.Listen(ctx); err != nil { - // Binding the Local port is the duplicate-launch guard, and now an - // atomic one: a second Client on the same port loses the bind instead - // of racing the first for the room authority. - var bindErr *net.OpError - if errors.As(err, &bindErr) && bindErr.Op == "listen" { - return fmt.Errorf("local server port is unavailable: %w", err) - } return err } - defer func() { _ = endLocalServer(localServer) }() + slog.Debug("screener-client", "event", "local-server-ready", "port", options.Port, "publicLink", publicOrigin != "") // The readiness lines follow the listener, which the packaged smoke and the // public-link gate both read from stdout before they probe the port. @@ -354,7 +445,7 @@ func runLocal(ctx context.Context, options Options, config clientconfig.Config, } } launchURL := clientLaunchURLWithLocalAccess( - fmt.Sprintf("http://localhost:%d/", options.Port), + fmt.Sprintf("http://127.0.0.1:%d/", options.Port), config.LocalAccessPassword, ) if !options.DisableBrowser { diff --git a/internal/client/clientapp/app_test.go b/internal/client/clientapp/app_test.go index 56ebb18..dfa761b 100644 --- a/internal/client/clientapp/app_test.go +++ b/internal/client/clientapp/app_test.go @@ -1,14 +1,38 @@ package clientapp import ( + "bytes" + "errors" + "log" + "log/slog" + "net" "net/url" + "os" + "path/filepath" "slices" + "strings" "testing" "github.com/TNTcraftHIM/Screener/internal/client/clientconfig" serverconfig "github.com/TNTcraftHIM/Screener/internal/server/config" ) +func TestOccupiedPortRejectsLinkBeforeStartingATunnel(t *testing.T) { + listener, err := net.ListenTCP("tcp4", &net.TCPAddr{IP: net.IPv4zero}) + if err != nil { + t.Fatal(err) + } + defer listener.Close() + err = runLocal(t.Context(), Options{ + Link: true, Port: listener.Addr().(*net.TCPAddr).Port, + TunnelProcess: "missing-tunnel-process", console: &clientConsole{machine: true}, + }, clientconfig.Config{}, nil) + var bindError *net.OpError + if !errors.As(err, &bindError) || bindError.Op != "listen" { + t.Fatalf("occupied port must fail before tunnel work: %v", err) + } +} + func TestClientLaunchURLMarksThePageWithoutChangingOrigin(t *testing.T) { value := clientLaunchURL("https://share.example/") parsed, err := url.Parse(value) @@ -22,6 +46,79 @@ func TestClientLaunchURLMarksThePageWithoutChangingOrigin(t *testing.T) { } } +func TestClientDiagnosticsUseFilesOnlyWhenEnabled(t *testing.T) { + for value, want := range map[string]bool{ + "client": true, "route, client": true, "client,": true, + "": false, "route": false, "all": false, "client-secret": false, + } { + if got := clientDebugEnabled(value); got != want { + t.Fatalf("clientDebugEnabled(%q) = %v, want %v", value, got, want) + } + } + + var output bytes.Buffer + previousOutput := log.Writer() + previousLevel := slog.SetLogLoggerLevel(slog.LevelInfo) + log.SetOutput(&output) + t.Cleanup(func() { + log.SetOutput(previousOutput) + slog.SetLogLoggerLevel(previousLevel) + }) + t.Setenv("SCREENER_DEBUG", "") + for _, enabled := range []bool{false, true} { + output.Reset() + directory := t.TempDir() + err := Run(t.Context(), Options{Debug: enabled, LogDir: directory, DisableBrowser: true, Local: true, Link: true}) + if err == nil { + t.Fatal("invalid mode must stop before starting capture or services") + } + if strings.Contains(output.String(), "screener-client") { + t.Fatalf("Client diagnostics reached stderr: %s", output.String()) + } + content, readErr := os.ReadFile(filepath.Join(directory, "client.log")) + if !enabled { + if !errors.Is(readErr, os.ErrNotExist) { + t.Fatalf("disabled diagnostics created a log: %v", readErr) + } + continue + } + if readErr != nil || !strings.Contains(string(content), `"event":"start"`) || + !strings.Contains(string(content), `"revision":`) || + !strings.Contains(string(content), `"event":"stopped","failed":true`) { + t.Fatalf("missing file lifecycle events: %s, %v", content, readErr) + } + } +} + +func TestClientDiagnosticDirectoryHonorsExplicitSelection(t *testing.T) { + environmentDirectory := t.TempDir() + t.Setenv("SCREENER_LOG_DIR", environmentDirectory) + for _, directory := range []string{"", t.TempDir()} { + recorder, err := openClientDiagnostics(directory) + if err != nil { + t.Fatal(err) + } + want := directory + if want == "" { + want = environmentDirectory + } + if recorder.LogPath() != filepath.Join(want, "client.log") { + t.Fatalf("diagnostic path = %q", recorder.LogPath()) + } + if err = recorder.Close(); err != nil { + t.Fatal(err) + } + } + blocked := filepath.Join(t.TempDir(), "not-a-directory") + if err := os.WriteFile(blocked, nil, 0o600); err != nil { + t.Fatal(err) + } + if recorder, err := openClientDiagnostics(blocked); err == nil { + _ = recorder.Close() + t.Fatal("an explicit directory failure silently fell back") + } +} + func TestMissingCaptureKeepsViewerControlAvailable(t *testing.T) { native := discoverNativeMedia(t.Context(), "missing-capture-process") if native.capabilities.Video || native.capabilities.HardwareH264 { diff --git a/internal/client/clientapp/console.go b/internal/client/clientapp/console.go index d6915cd..e8ed976 100644 --- a/internal/client/clientapp/console.go +++ b/internal/client/clientapp/console.go @@ -7,6 +7,7 @@ import ( "math/rand/v2" "net/url" "os" + "path/filepath" "strings" "sync" "time" @@ -31,6 +32,14 @@ type consoleTick struct{} type consoleIdleWink struct{} type consoleOpenResult struct{ err error } type consoleDiagnostic string +type consoleDebug struct { + logPath string + export func() (string, error) +} +type consoleExportResult struct { + path string + err error +} type consoleModel struct { view consoleView @@ -44,6 +53,10 @@ type consoleModel struct { idleWink bool finished bool cancel context.CancelFunc + debug consoleDebug + exporting bool + exportPath string + exportError bool } type clientConsole struct { @@ -56,24 +69,29 @@ type clientConsole struct { } var consoleCopy = map[string][3]string{ - "mode": {"Mode", "模式", ""}, - "local": {"Local network", "局域网", "□"}, - "link": {"Public link", "公网链接", "↗"}, - "site": {"Screener Site", "Screener 站点", "@"}, - "setup": {"Choose a mode in the browser", "在浏览器中选择模式", "?"}, - "starting": {"Starting", "正在启动", "…"}, - "ready": {"Ready", "已就绪", "✓"}, - "stopping": {"Stopping", "正在退出", "→"}, - "stopped": {"Stopped", "已停止", "○"}, - "failed": {"Could not continue", "运行失败", "!"}, - "entry": {"Open in browser", "打开网页", "□"}, - "invite": {"Site address", "站点地址", "↗"}, - "access": {"Site access", "站点准入", ""}, - "open": {"Open", "开放", "○"}, - "password": {"Password protected", "已设置密码", "*"}, - "error": {"Details", "详情", "!"}, - "exit": {"quit", "退出", "↪"}, - "openFailed": {"Could not open the browser; use the address above.", "无法打开浏览器，请使用上方地址。", "! ↗"}, + "mode": {"Mode", "模式", ""}, + "local": {"Local network", "局域网", "□"}, + "link": {"Public link", "公网链接", "↗"}, + "site": {"Screener Site", "Screener 站点", "@"}, + "setup": {"Choose a mode in the browser", "在浏览器中选择模式", "?"}, + "starting": {"Starting", "正在启动", "…"}, + "ready": {"Ready", "已就绪", "✓"}, + "stopping": {"Stopping", "正在退出", "→"}, + "stopped": {"Stopped", "已停止", "○"}, + "failed": {"Could not continue", "运行失败", "!"}, + "entry": {"Open in browser", "打开网页", "□"}, + "invite": {"Site address", "站点地址", "↗"}, + "access": {"Site access", "站点准入", ""}, + "open": {"Open", "开放", "○"}, + "password": {"Password protected", "已设置密码", "*"}, + "error": {"Details", "详情", "!"}, + "exit": {"quit", "退出", "↪"}, + "openFailed": {"Could not open the browser; use the address above.", "无法打开浏览器，请使用上方地址。", "! ↗"}, + "log": {"Log", "日志", ""}, + "export": {"Export diagnostics", "导出诊断", ""}, + "exporting": {"Saving diagnostics", "正在保存诊断", ""}, + "exported": {"Saved", "已保存", ""}, + "exportFailed": {"Could not save diagnostics", "无法保存诊断", ""}, } func newClientConsole(cancel context.CancelFunc, machine bool) *clientConsole { @@ -159,6 +177,13 @@ func (model consoleModel) Update(message tea.Msg) (tea.Model, tea.Cmd) { model.language = string(value) case consoleDiagnostic: model.view.problem = string(value) + case consoleDebug: + model.debug = value + case consoleExportResult: + model.exporting, model.exportError = false, value.err != nil + if value.err == nil { + model.exportPath = value.path + } case tea.WindowSizeMsg: model.width, model.height = value.Width, value.Height case tea.ColorProfileMsg: @@ -195,6 +220,14 @@ func (model consoleModel) Update(message tea.Msg) (tea.Model, tea.Cmd) { if target := model.view.entry; target != "" { return model, func() tea.Msg { return consoleOpenResult{browser.Open(clientLaunchURL(consoleAddress(target)))} } } + case "d", "D": + if model.debug.export != nil && !model.exporting && !model.finished && model.view.state != "stopping" { + model.exporting, model.exportError = true, false + return model, func() tea.Msg { + path, err := model.debug.export() + return consoleExportResult{path: path, err: err} + } + } } } if model.animating() && !model.tickPending { @@ -235,6 +268,7 @@ func (model consoleModel) View() tea.View { func (model consoleModel) content(styled bool) string { width := max(12, min(model.width-2, 74)) compact := model.height > 0 && model.height < 24 + compactDebug := compact && model.debug.logPath != "" visual := model.language == "vis" accent, muted, link := lipgloss.NewStyle(), lipgloss.NewStyle(), lipgloss.NewStyle() if styled { @@ -250,7 +284,7 @@ func (model consoleModel) content(styled bool) string { heading := "Screener\nClient\n" + muted.Render(revision) animated := styled && model.animating() mascot := accent.Render(consoleTV(model.frame, animated, compact || width < 34)) - if styled && model.colors && width >= 36 { + if styled && model.colors && width >= 36 && !compactDebug { mascot = consoleBlockTV(model.frame, animated) } fmt.Fprintln(&out, lipgloss.JoinHorizontal(lipgloss.Center, mascot, heading)) @@ -280,6 +314,8 @@ func (model consoleModel) content(styled bool) string { fmt.Fprintln(&out, lipgloss.JoinHorizontal(lipgloss.Center, accent.Render(consoleVisualToken(model.view.mode)), " · ", stateStyle.Render(consoleVisualToken(model.view.state)))) + } else if compactDebug && model.view.mode != "" { + fmt.Fprintf(&out, "%s %s\n", accent.Render(state), model.text(model.view.mode)) } else { fmt.Fprintln(&out, accent.Render(state)) if model.view.mode != "" { @@ -299,7 +335,9 @@ func (model consoleModel) content(styled bool) string { } style = style.Hyperlink(target) } - if visual { + if compactDebug { + fmt.Fprintf(&out, "\n%s\n", style.Render(address)) + } else if visual { label := consoleVisualToken(item[0]) fmt.Fprintf(&out, "\n%s %s\n", label, style.Render(address)) } else { @@ -325,6 +363,22 @@ func (model consoleModel) content(styled bool) string { } fmt.Fprintf(&out, "\n%s\n%s\n", model.text("error"), strings.Join(lines, "\n")) } + if model.debug.logPath != "" { + path, label := model.debug.logPath, "log" + if model.exportPath != "" { + path, label = model.exportPath, "exported" + } + separator := "\n" + if compactDebug { + separator = " " + } + fmt.Fprintf(&out, "\n%s%s%s\n", muted.Render(model.text(label)), separator, consoleFileLink(path, path, link, styled)) + if model.exporting { + fmt.Fprintln(&out, model.text("exporting")) + } else if model.exportError { + fmt.Fprintln(&out, model.text("exportFailed")) + } + } if !model.finished { keys := "[q / Ctrl+C]" if !styled { @@ -337,7 +391,14 @@ func (model consoleModel) content(styled bool) string { } help = lipgloss.JoinHorizontal(lipgloss.Center, accessPicture, help) } else if model.view.entry != "" && styled { - help = "[o] " + model.text("entry") + " " + help + separator := " " + if compact && model.debug.export != nil { + separator = "\n" + } + help = "[o] " + model.text("entry") + separator + help + } + if model.debug.export != nil && styled { + help = "[d] " + model.text("export") + "\n" + help } if visual { fmt.Fprintf(&out, "\n%s", accent.Render(help)) @@ -388,6 +449,16 @@ func consoleVisualToken(key string) string { return "↗" case "exit": return "↪" + case "log": + return "LOG" + case "export": + return "ZIP" + case "exporting": + return "ZIP ..." + case "exported": + return "ZIP +" + case "exportFailed": + return "ZIP !" default: return "?" } @@ -481,3 +552,15 @@ func consoleAddress(raw string) string { parsed.User, parsed.RawQuery, parsed.Fragment, parsed.RawFragment, parsed.ForceQuery = nil, "", "", "", false return parsed.String() } + +func consoleFileLink(path, label string, style lipgloss.Style, styled bool) string { + if !styled { + return label + } + path = filepath.ToSlash(path) + if !strings.HasPrefix(path, "/") { + path = "/" + path + } + address := url.URL{Scheme: "file", Path: path} + return style.Hyperlink(address.String()).Render(label) +} diff --git a/internal/client/clientapp/console_test.go b/internal/client/clientapp/console_test.go index d94ffd5..0f71049 100644 --- a/internal/client/clientapp/console_test.go +++ b/internal/client/clientapp/console_test.go @@ -3,6 +3,8 @@ package clientapp import ( "bytes" "context" + "errors" + "path/filepath" "strings" "testing" "time" @@ -108,3 +110,75 @@ func TestConsolePresentationAndShutdown(t *testing.T) { t.Fatal("stopped console retained a live entry") } } + +func TestConsoleDiagnosticExportRunsOutsideUpdate(t *testing.T) { + called := 0 + path := filepath.Join(t.TempDir(), "client-report.zip") + model := consoleModel{ + view: consoleView{state: "ready"}, frame: 16, idlePending: true, + debug: consoleDebug{logPath: filepath.Join(filepath.Dir(path), "client.log"), export: func() (string, error) { + called++ + return path, nil + }}, + } + updated, command := model.Update(tea.KeyPressMsg{Code: 'd'}) + model = updated.(consoleModel) + if called != 0 || command == nil || !model.exporting { + t.Fatal("diagnostic export blocked the console update") + } + _, repeated := model.Update(tea.KeyPressMsg{Code: 'd'}) + if repeated != nil { + t.Fatal("repeated key queued another diagnostic export") + } + updated, _ = model.Update(command()) + model = updated.(consoleModel) + if called != 1 || model.exporting || model.exportPath != path || model.exportError { + t.Fatalf("diagnostic result = %+v", model) + } + updated, _ = model.Update(consoleExportResult{err: errors.New("disk full")}) + model = updated.(consoleModel) + if !model.exportError || model.exportPath != path { + t.Fatal("failed export discarded the previously saved bundle") + } + for _, language := range []string{"en", "zh", "vis"} { + for _, width := range []int{30, 52, 90} { + model.language, model.width, model.height = language, width, 40 + for _, styled := range []bool{false, true} { + content := model.content(styled) + for _, line := range strings.Split(content, "\n") { + if ansi.StringWidth(line) > width { + t.Fatalf("%s diagnostic console width %d overflowed: %q", language, width, line) + } + } + if styled && !strings.Contains(content, "[d]") || !styled && strings.Contains(content, "[d]") { + t.Fatalf("export key hint does not match terminal input: %q", content) + } + } + } + } + link := consoleFileLink(path, path, lipgloss.NewStyle(), true) + if !strings.Contains(link, "file:///") || strings.Contains(link, "file:///C:\\") { + t.Fatalf("local report hyperlink = %q", link) + } +} + +func TestConsoleDiagnosticsFitCompactTerminal(t *testing.T) { + model := consoleModel{ + view: consoleView{mode: "local", state: "ready", entry: "http://localhost:8787/"}, + width: 30, height: 18, colors: true, + debug: consoleDebug{logPath: "C:/Screener/logs/client.log", export: func() (string, error) { return "", nil }}, + } + for _, language := range []string{"en", "zh", "vis"} { + model.language = language + for _, path := range []string{"", "C:/Screener/logs/client-20260908T120000Z.zip", "C:/Users/Example/Documents/Workspaces/Screener-embedded-media/build/client-check/logs/client-diagnostics-20260908T120000Z-123456789.zip"} { + model.exportPath = path + content := model.content(true) + if lipgloss.Width(content) > model.width || lipgloss.Height(content) > model.height { + t.Fatalf("%s diagnostic console overflowed: %s", language, content) + } + if !strings.Contains(content, "[d]") || !strings.Contains(content, "[o]") || !strings.Contains(content, "[q") { + t.Fatalf("compact diagnostics hid a terminal action: %s", content) + } + } + } +} diff --git a/internal/client/loopback/protocol.go b/internal/client/loopback/protocol.go index 321b53e..530d341 100644 --- a/internal/client/loopback/protocol.go +++ b/internal/client/loopback/protocol.go @@ -9,9 +9,9 @@ import ( ) const ( - ProtocolVersion = 8 + ProtocolVersion = 9 ServiceName = "screener-client" - ControlSubprotocol = "screener-client-v8" + ControlSubprotocol = "screener-client-v9" MaxControlMessageBytes = 256 << 10 DefaultPortStart = 39721 DefaultPortEnd = 39730 diff --git a/internal/client/loopback/protocol_test.go b/internal/client/loopback/protocol_test.go index 7b89f52..5caf5ca 100644 --- a/internal/client/loopback/protocol_test.go +++ b/internal/client/loopback/protocol_test.go @@ -7,8 +7,8 @@ import ( func TestDecodeRequestAcceptsOnlyTheCurrentShape(t *testing.T) { valid := []string{ - `{"version":8,"id":"request_hello","type":"hello"}`, - `{"version":8,"id":"request_ping","type":"ping"}`, + `{"version":9,"id":"request_hello","type":"hello"}`, + `{"version":9,"id":"request_ping","type":"ping"}`, } for _, payload := range valid { if _, err := decodeRequest([]byte(payload)); err != nil { @@ -17,10 +17,10 @@ func TestDecodeRequestAcceptsOnlyTheCurrentShape(t *testing.T) { } invalid := []string{ `{"version":3,"id":"request_ping","type":"ping"}`, - `{"version":8,"id":"short","type":"ping"}`, - `{"version":8,"id":"request_ping","type":"ping","extra":true}`, - `{"version":8,"id":"request_hello","type":"hello","nonce":"obsolete"}`, - `{"version":8,"id":"request_ping","type":"ping"} trailing`, + `{"version":9,"id":"short","type":"ping"}`, + `{"version":9,"id":"request_ping","type":"ping","extra":true}`, + `{"version":9,"id":"request_hello","type":"hello","nonce":"obsolete"}`, + `{"version":9,"id":"request_ping","type":"ping"} trailing`, } for _, payload := range invalid { if _, err := decodeRequest([]byte(payload)); err == nil { @@ -34,11 +34,11 @@ func TestDecodeRequestUsesTheSharedIdentifierBoundary(t *testing.T) { if len(identifier) != 256 { t.Fatalf("test identifier length = %d", len(identifier)) } - payload := []byte(`{"version":8,"id":"` + identifier + `","type":"ping"}`) + payload := []byte(`{"version":9,"id":"` + identifier + `","type":"ping"}`) if _, err := decodeRequest(payload); err != nil { t.Fatalf("maximum identifier was rejected: %v", err) } - tooLong := []byte(`{"version":8,"id":"` + identifier + `a","type":"ping"}`) + tooLong := []byte(`{"version":9,"id":"` + identifier + `a","type":"ping"}`) if _, err := decodeRequest(tooLong); err == nil { t.Fatal("identifier beyond the boundary was accepted") } @@ -62,7 +62,7 @@ func TestValidateMessagesRequireTheExpectedPhase(t *testing.T) { } func TestEnvelopeAllowsAnExtensionToOwnItsStrictShape(t *testing.T) { - payload := []byte(`{"version":8,"id":"request_extension","type":"extension","value":1}`) + payload := []byte(`{"version":9,"id":"request_extension","type":"extension","value":1}`) message, err := decodeEnvelope(payload) if err != nil || message.Type != "extension" { t.Fatalf("extension envelope = %+v, %v", message, err) diff --git a/internal/client/loopback/server.go b/internal/client/loopback/server.go index 02ecb91..2632a35 100644 --- a/internal/client/loopback/server.go +++ b/internal/client/loopback/server.go @@ -25,6 +25,7 @@ const ( helloTimeout = 5 * time.Second shutdownTimeout = 2 * time.Second controlQueueSize = 64 // MaxControlMessageBytes * 64 = 16 MiB worst-case. + maxControlSessions = 2 ) type Options struct { @@ -37,6 +38,7 @@ type Options struct { } type ControlSession interface { + // A nil response with no error is completed later through Events, using the request ID. Handle(context.Context, []byte) (any, error) Events() <-chan any Close() error @@ -77,10 +79,13 @@ type Server struct { presentation func(string) done chan error - mu sync.Mutex - activeConn *websocket.Conn - claimed bool - closed bool + mu sync.Mutex + connections map[*websocket.Conn]struct{} + claimed int + closed bool + controls sync.WaitGroup + closeOnce sync.Once + closeErr error } func Start(parent context.Context, options Options) (*Server, error) { @@ -116,6 +121,7 @@ func Start(parent context.Context, options Options) (*Server, error) { newControl: options.NewControl, presentation: options.Presentation, done: make(chan error, 1), + connections: make(map[*websocket.Conn]struct{}), } server.httpServer = &http.Server{ Handler: server, @@ -145,23 +151,33 @@ func (server *Server) SetAllowedOrigins(origins []string) { } func (server *Server) Close() error { - server.mu.Lock() - if server.closed { + server.closeOnce.Do(func() { + server.mu.Lock() + server.closed = true + connections := make([]*websocket.Conn, 0, len(server.connections)) + for connection := range server.connections { + connections = append(connections, connection) + } server.mu.Unlock() - return nil - } - server.closed = true - connection := server.activeConn - server.activeConn = nil - server.claimed = false - server.mu.Unlock() - server.cancel() - if connection != nil { - _ = connection.Close(websocket.StatusNormalClosure, "helper stopped") - } - ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) - defer cancel() - return server.httpServer.Shutdown(ctx) + server.cancel() + ctx, cancel := context.WithTimeout(context.Background(), shutdownTimeout) + defer cancel() + for _, connection := range connections { + _ = connection.CloseNow() + } + server.closeErr = server.httpServer.Shutdown(ctx) + retired := make(chan struct{}) + go func() { + server.controls.Wait() + close(retired) + }() + select { + case <-retired: + case <-ctx.Done(): + server.closeErr = errors.Join(server.closeErr, ctx.Err()) + } + }) + return server.closeErr } func (server *Server) serve() { @@ -196,6 +212,7 @@ func (server *Server) handlePresentation(response http.ResponseWriter, request * if request.Method == http.MethodOptions { response.Header().Set("Access-Control-Allow-Methods", http.MethodPost) response.Header().Set("Access-Control-Allow-Headers", "Content-Type") + allowPrivateNetwork(response, request) response.WriteHeader(http.StatusNoContent) return } @@ -222,8 +239,14 @@ func (server *Server) handlePresentation(response http.ResponseWriter, request * } func (server *Server) handleHealth(response http.ResponseWriter, request *http.Request) { + if request.Method == http.MethodOptions { + response.Header().Set("Access-Control-Allow-Methods", http.MethodGet) + allowPrivateNetwork(response, request) + response.WriteHeader(http.StatusNoContent) + return + } if request.Method != http.MethodGet { - response.Header().Set("Allow", http.MethodGet) + response.Header().Set("Allow", "GET, OPTIONS") http.Error(response, "method not allowed", http.StatusMethodNotAllowed) return } @@ -237,6 +260,12 @@ func (server *Server) handleHealth(response http.ResponseWriter, request *http.R }) } +func allowPrivateNetwork(response http.ResponseWriter, request *http.Request) { + if strings.EqualFold(strings.TrimSpace(request.Header.Get("Access-Control-Request-Private-Network")), "true") { + response.Header().Set("Access-Control-Allow-Private-Network", "true") + } +} + func (server *Server) handleControl(response http.ResponseWriter, request *http.Request) { if request.Method != http.MethodGet { response.Header().Set("Allow", http.MethodGet) @@ -244,26 +273,28 @@ func (server *Server) handleControl(response http.ResponseWriter, request *http. return } if !server.claimConnection() { - http.Error(response, "control session already active", http.StatusConflict) + http.Error(response, "native control capacity is occupied", http.StatusConflict) return } + var connection *websocket.Conn + defer func() { server.releaseConnection(connection) }() expectedSubprotocol := ControlSubprotocol + "." + server.endpoint.InstanceToken - connection, err := websocket.Accept(response, request, &websocket.AcceptOptions{ + var err error + connection, err = websocket.Accept(response, request, &websocket.AcceptOptions{ Subprotocols: []string{expectedSubprotocol}, OriginPatterns: server.originPatterns(), }) if err != nil { - server.releaseConnection(nil) return } + defer connection.CloseNow() if connection.Subprotocol() != expectedSubprotocol { closeControl(connection, "unsupported control protocol") - server.releaseConnection(connection) return } - server.setConnection(connection) - defer server.releaseConnection(connection) - defer connection.CloseNow() + if !server.setConnection(connection) { + return + } connection.SetReadLimit(MaxControlMessageBytes) helloContext, cancelHello := context.WithTimeout(server.ctx, helloTimeout) @@ -368,10 +399,13 @@ func (server *Server) handleControl(response http.ResponseWriter, request *http. return } value, handleErr := extension.Handle(controlContext, payload) - if handleErr != nil || value == nil { + if handleErr != nil { closeControl(connection, "invalid control message") return } + if value == nil { + continue + } if err = write(controlContext, value); err != nil { return } @@ -443,26 +477,30 @@ func (server *Server) localOrigin(origin string) bool { func (server *Server) claimConnection() bool { server.mu.Lock() defer server.mu.Unlock() - if server.closed || server.claimed { + if server.closed || server.claimed >= maxControlSessions { return false } - server.claimed = true + server.claimed++ + server.controls.Add(1) return true } -func (server *Server) setConnection(connection *websocket.Conn) { +func (server *Server) setConnection(connection *websocket.Conn) bool { server.mu.Lock() - server.activeConn = connection - server.mu.Unlock() + defer server.mu.Unlock() + if server.closed { + return false + } + server.connections[connection] = struct{}{} + return true } func (server *Server) releaseConnection(connection *websocket.Conn) { server.mu.Lock() - if server.activeConn == connection { - server.activeConn = nil - } - server.claimed = false + delete(server.connections, connection) + server.claimed-- server.mu.Unlock() + server.controls.Done() } func listenInRange(bindAddress string, start, end int) (net.Listener, error) { diff --git a/internal/client/loopback/server_test.go b/internal/client/loopback/server_test.go index eba0503..40fe4b1 100644 --- a/internal/client/loopback/server_test.go +++ b/internal/client/loopback/server_test.go @@ -116,7 +116,8 @@ func TestPresentationUpdatesOnlyLanguageWithoutClaimingMediaControl(t *testing.T {http.MethodOptions, "/presentation", testOrigin, "", "", http.StatusNoContent}, {http.MethodPost, "/presentation", "https://other.example", `{"language":"en"}`, "", http.StatusForbidden}, {http.MethodOptions, "/presentation", "https://other.example", "", "", http.StatusForbidden}, - {http.MethodOptions, "/health", testOrigin, "", "", http.StatusMethodNotAllowed}, + {http.MethodOptions, "/health", testOrigin, "", "", http.StatusNoContent}, + {http.MethodOptions, "/health", "https://other.example", "", "", http.StatusForbidden}, {http.MethodGet, "/presentation", testOrigin, "", "", http.StatusMethodNotAllowed}, {http.MethodPost, "/presentation", testOrigin, `{"language":"other"}`, "", http.StatusBadRequest}, {http.MethodPost, "/presentation", testOrigin, `{"language":"en","capture":true}`, "", http.StatusBadRequest}, @@ -129,6 +130,9 @@ func TestPresentationUpdatesOnlyLanguageWithoutClaimingMediaControl(t *testing.T } request.Header.Set("Origin", input.origin) request.Header.Set("Content-Type", "application/json") + if input.method == http.MethodOptions { + request.Header.Set("Access-Control-Request-Private-Network", "true") + } response, err := client.Do(request) if err != nil { t.Fatal(err) @@ -139,8 +143,10 @@ func TestPresentationUpdatesOnlyLanguageWithoutClaimingMediaControl(t *testing.T } if input.method == http.MethodOptions && input.status == http.StatusNoContent && (response.Header.Get("Access-Control-Allow-Origin") != testOrigin || - response.Header.Get("Access-Control-Allow-Methods") != "POST" || - response.Header.Get("Access-Control-Allow-Headers") != "Content-Type") { + response.Header.Get("Access-Control-Allow-Private-Network") != "true" || + (input.path == "/presentation" && (response.Header.Get("Access-Control-Allow-Methods") != "POST" || + response.Header.Get("Access-Control-Allow-Headers") != "Content-Type")) || + (input.path == "/health" && response.Header.Get("Access-Control-Allow-Methods") != "GET")) { t.Fatal("presentation preflight did not preserve its bounded origin/method scope") } select { @@ -162,20 +168,65 @@ func TestPresentationUpdatesOnlyLanguageWithoutClaimingMediaControl(t *testing.T } } -func TestOnlyOneControlSessionIsClaimed(t *testing.T) { - server := startTestServer(t, testOrigin) +func TestControlSessionsAreBoundedAndRetireIndependently(t *testing.T) { + created := make(chan *testControlSession, maxControlSessions) + server := startTestServerWithOptions(t, Options{ + AllowedOrigins: []string{testOrigin}, + NewControl: func() ControlSession { + session := &testControlSession{events: make(chan any), closed: make(chan struct{})} + created <- session + return session + }, + }) endpoint := server.Endpoint() - first := dialControl(t, endpoint, endpoint.InstanceToken, testOrigin) - defer first.CloseNow() - second, response, err := websocket.Dial(context.Background(), websocketURL(endpoint)+"/control", &websocket.DialOptions{ + var connections []*websocket.Conn + var sessions []*testControlSession + for index := 0; index < maxControlSessions; index++ { + connection := dialControl(t, endpoint, endpoint.InstanceToken, testOrigin) + defer connection.CloseNow() + writeControl(t, connection, requestJSON("request_hello", "hello")) + var ready controlMessage + readControl(t, connection, &ready) + connections = append(connections, connection) + select { + case session := <-created: + sessions = append(sessions, session) + case <-time.After(time.Second): + t.Fatal("accepted control did not acquire its own session") + } + } + extra, response, err := websocket.Dial(context.Background(), websocketURL(endpoint)+"/control", &websocket.DialOptions{ HTTPHeader: http.Header{"Origin": []string{testOrigin}}, Subprotocols: []string{ControlSubprotocol + "." + endpoint.InstanceToken}, }) - if second != nil { - second.CloseNow() + if extra != nil { + extra.CloseNow() } if err == nil || response == nil || response.StatusCode != http.StatusConflict { - t.Fatalf("second control dial: conn=%v response=%v err=%v", second, response, err) + t.Fatalf("excess control dial: conn=%v response=%v err=%v", extra, response, err) + } + _ = response.Body.Close() + _ = connections[0].CloseNow() + select { + case <-sessions[0].closed: + case <-time.After(time.Second): + t.Fatal("closed room retained its native session") + } + writeControl(t, connections[1], requestJSON("request_ping", "ping")) + var pong controlMessage + readControl(t, connections[1], &pong) + if pong.Type != "pong" { + t.Fatal("closing one room affected its sibling control") + } + if err = server.Close(); err != nil { + t.Fatal(err) + } + for _, session := range sessions { + select { + case <-session.closed: + default: + t.Fatal("Client close returned before native session cleanup") + } } } @@ -316,6 +367,28 @@ func TestControlSessionProcessesARequestBurstInOrder(t *testing.T) { } } +func TestDeferredResponseKeepsControlResponsiveAndRequestIdentity(t *testing.T) { + extension := &testControlSession{events: make(chan any, 1), closed: make(chan struct{})} + server := startTestServerWithOptions(t, Options{AllowedOrigins: []string{testOrigin}, + NewControl: func() ControlSession { return extension }}) + connection := dialControl(t, server.Endpoint(), server.Endpoint().InstanceToken, testOrigin) + defer connection.CloseNow() + writeControl(t, connection, requestJSON("request_hello", "hello")) + var value controlMessage + readControl(t, connection, &value) + writeControl(t, connection, requestJSON("request_pending", "deferred")) + writeControl(t, connection, requestJSON("request_ping", "ping")) + readControl(t, connection, &value) + if value.Type != "pong" || value.ID != "request_ping" { + t.Fatalf("pending operation blocked control: %+v", value) + } + extension.events <- controlMessage{Version: ProtocolVersion, ID: "request_pending", Type: "extension-response"} + readControl(t, connection, &value) + if value.Type != "extension-response" || value.ID != "request_pending" { + t.Fatalf("deferred response lost its request identity: %+v", value) + } +} + func TestStartAlwaysBindsToIPv4Loopback(t *testing.T) { server := startTestServer(t, testOrigin) host, _, err := net.SplitHostPort(server.Endpoint().Host) @@ -468,6 +541,9 @@ func (session *blockingControlSession) Close() error { func (session *testControlSession) Handle(_ context.Context, payload []byte) (any, error) { message, err := decodeEnvelope(payload) + if err == nil && message.Type == "deferred" { + return nil, nil + } if err != nil || message.Type != "extension" { return nil, errors.New("unexpected extension request") } diff --git a/internal/client/mediaedge/audio.go b/internal/client/mediaedge/audio.go index 2b8dec3..c8298b0 100644 --- a/internal/client/mediaedge/audio.go +++ b/internal/client/mediaedge/audio.go @@ -140,7 +140,28 @@ func (source *AudioSource) SetBitrate(bitrate int) error { if closed { return errors.New("native audio source is closed") } - return source.encoder.SetBitrate(bitrate) + if err := source.encoder.SetBitrate(bitrate); err != nil { + return err + } + source.mu.Lock() + edges := make([]*Edge, 0, len(source.edges)) + for edge := range source.edges { + edges = append(edges, edge) + } + source.mu.Unlock() + for _, edge := range edges { + edge.transport.SetAudioBitrate(uint32(bitrate)) + } + return nil +} + +func (source *AudioSource) configuredBitrate() uint32 { + if source == nil || source.encoder == nil { + return 0 + } + source.encodeMu.Lock() + defer source.encodeMu.Unlock() + return uint32(source.encoder.Bitrate()) } func (source *AudioSource) snapshotBytes() uint64 { diff --git a/internal/client/mediaedge/edge.go b/internal/client/mediaedge/edge.go index 6bea973..6e96b85 100644 --- a/internal/client/mediaedge/edge.go +++ b/internal/client/mediaedge/edge.go @@ -2,13 +2,15 @@ package mediaedge import ( "errors" + "fmt" "io" + "log/slog" "strconv" "strings" "sync" + "github.com/TNTcraftHIM/Screener/internal/media/forwarding" "github.com/pion/ice/v4" - "github.com/pion/rtcp" "github.com/pion/webrtc/v4" ) @@ -28,15 +30,17 @@ type EdgeOptions struct { } type Edge struct { - connectionID string - engine *Engine - source *Source - audioSource *AudioSource - connection *webrtc.PeerConnection - sender *webrtc.RTPSender - audioSender *webrtc.RTPSender - bandwidth *bandwidthObserver - events EdgeEvents + connectionID string + engine *Engine + source *Source + audioSource *AudioSource + connection *webrtc.PeerConnection + sender *webrtc.RTPSender + audioSender *webrtc.RTPSender + targetBitrate func() (int, bool) + events EdgeEvents + transport *forwarding.Transport + local bool mu sync.Mutex pendingCandidates []webrtc.ICECandidateInit @@ -69,7 +73,14 @@ func (engine *Engine) NewEdge(source *Source, options EdgeOptions) (*Edge, error return nil, err } } - connection, err := engine.api.NewPeerConnection(webrtc.Configuration{}) + var audio webrtc.TrackLocal + if options.Audio != nil { + audio = options.Audio.track + } + transport, err := forwarding.NewTransport(forwarding.TransportOptions{ + Source: source.media.Source, Settings: engine.settings, ConnectionID: options.ConnectionID, + InitialBitrate: engine.initialBitrate, Audio: audio, + }) if err != nil { source.releaseReservation(options.Local) if options.Audio != nil { @@ -77,30 +88,27 @@ func (engine *Engine) NewEdge(source *Source, options EdgeOptions) (*Edge, error } return nil, err } - bandwidth := engine.bandwidth.take(connection.ID()) - if bandwidth == nil { - source.releaseReservation(options.Local) - if options.Audio != nil { - options.Audio.releaseReservation(options.Local) - } - _ = connection.Close() - return nil, errors.New("native media bandwidth observer is unavailable") - } + connection := transport.PC + transport.SetAudioBitrate(options.Audio.configuredBitrate()) edge := &Edge{ - connectionID: options.ConnectionID, - engine: engine, - source: source, - audioSource: options.Audio, - connection: connection, - bandwidth: bandwidth, - events: options.Events, + connectionID: options.ConnectionID, + engine: engine, + source: source, + audioSource: options.Audio, + connection: connection, + transport: transport, + local: options.Local, + targetBitrate: transport.TargetBitrate, + events: options.Events, + sender: transport.VideoSender, + audioSender: transport.AudioSender, } if err = engine.register(edge); err != nil { source.releaseReservation(options.Local) if options.Audio != nil { options.Audio.releaseReservation(options.Local) } - _ = connection.Close() + _ = transport.Close() return nil, err } if err = source.attach(edge, options.Local); err != nil { @@ -108,39 +116,17 @@ func (engine *Engine) NewEdge(source *Source, options EdgeOptions) (*Edge, error options.Audio.releaseReservation(options.Local) } engine.remove(edge) - _ = connection.Close() + _ = transport.Close() return nil, err } if options.Audio != nil { if err = options.Audio.attach(edge, options.Local); err != nil { source.detach(edge) engine.remove(edge) - _ = connection.Close() + _ = transport.Close() return nil, err } } - sender, err := connection.AddTrack(source.track) - if err != nil { - _ = edge.Close() - return nil, err - } - edge.sender = sender - for _, transceiver := range connection.GetTransceivers() { - if transceiver.Sender() == sender { - if err = transceiver.SetCodecPreferences([]webrtc.RTPCodecParameters{videoCodecs[source.codec]}); err != nil { - _ = edge.Close() - return nil, err - } - } - } - if options.Audio != nil { - audioSender, audioErr := connection.AddTrack(options.Audio.track) - if audioErr != nil { - _ = edge.Close() - return nil, audioErr - } - edge.audioSender = audioSender - } edge.localCandidates = newLocalCandidateGathering( engine, options.ICEServers, @@ -150,24 +136,43 @@ func (engine *Engine) NewEdge(source *Source, options EdgeOptions) (*Edge, error connection.OnICECandidate(func(candidate *webrtc.ICECandidate) { edge.localCandidates.addPion(candidate) }) + connection.OnICEConnectionStateChange(func(state webrtc.ICEConnectionState) { + slog.Debug("screener-client", "event", "media-ice-state", "direction", "outbound", "local", options.Local, "state", state.String()) + }) + if dtls := edge.sender.Transport(); dtls != nil { + dtls.OnStateChange(func(state webrtc.DTLSTransportState) { + slog.Debug("screener-client", "event", "media-dtls-state", "direction", "outbound", "local", options.Local, "state", state.String()) + }) + } connection.OnConnectionStateChange(func(state webrtc.PeerConnectionState) { + debug := slog.Default().Enabled(engine.ctx, slog.LevelDebug) + if debug { + slog.Debug("screener-client", "event", "media-connection-state", "direction", "outbound", "local", options.Local, "state", state.String()) + } if state == webrtc.PeerConnectionStateConnected { + if err := edge.transport.SetConnected(); err != nil { + if debug { + slog.Debug("screener-client", "event", "media-activation-failed", "errorType", fmt.Sprintf("%T", err)) + } + _ = connection.Close() + return + } edge.source.RequestRecoveryFrame() } - if edge.events.ConnectionState != nil { - var selected *SelectedPair - if state == webrtc.PeerConnectionStateConnected { - if pair, pairErr := edge.SelectedPair(); pairErr == nil { - selected = &pair + var selected *SelectedPair + if state == webrtc.PeerConnectionStateConnected && (edge.events.ConnectionState != nil || debug) { + if pair, pairErr := edge.SelectedPair(); pairErr == nil { + selected = &pair + if debug { + slog.Debug("screener-client", "event", "media-selected-path", "direction", "outbound", "local", options.Local, + "localType", pair.Local.String(), "remoteType", pair.Remote.String(), "natTraversalPath", pair.NatTraversalPath) } } + } + if edge.events.ConnectionState != nil { edge.events.ConnectionState(state, selected) } }) - go edge.readRTCP(edge.sender, edge.source) - if edge.audioSender != nil { - go edge.readRTCP(edge.audioSender, nil) - } return edge, nil } @@ -175,6 +180,35 @@ func (edge *Edge) ConnectionID() string { return edge.connectionID } +// SetTargetLayer is a media-component input, not a route change or UI setting. +func (edge *Edge) SetTargetLayer(layer int) error { + source := edge.source + changed := false + source.writeMu.Lock() + defer func() { + source.writeMu.Unlock() + if changed { + source.RequestRecoveryFrame() + } + }() + source.mu.Lock() + _, attached := source.edges[edge] + closed := source.closed + source.mu.Unlock() + if closed || !attached { + return io.ErrClosedPipe + } + if layer < 0 || layer >= len(source.formats) { + return errors.New("native output layer is invalid") + } + if len(source.outputBitrates) > 0 && source.outputBitrates[layer] == 0 { + return errors.New("native output layer is unavailable") + } + changed = edge.transport.Output.MaxLayer().Spatial != int32(layer) + edge.transport.Output.SetMaxSpatialLayer(int32(layer)) + return nil +} + func (edge *Edge) CreateOffer() (webrtc.SessionDescription, error) { offer, err := edge.connection.CreateOffer(nil) if err != nil { @@ -318,30 +352,46 @@ func (edge *Edge) Close() error { } edge.pendingCandidates = nil edge.mu.Unlock() + if edge.local && slog.Default().Enabled(edge.engine.ctx, slog.LevelDebug) { + logLocalMediaCandidateCounts(edge.connection.GetStats()) + } + err := edge.transport.Close() edge.source.detach(edge) if edge.audioSource != nil { edge.audioSource.detach(edge) } edge.engine.remove(edge) - return edge.connection.Close() + return err } -func (edge *Edge) readRTCP(sender *webrtc.RTPSender, videoSource *Source) { - for { - packets, _, err := sender.ReadRTCP() - if err != nil { - if !errors.Is(err, io.EOF) { - return - } - return - } - for _, packet := range packets { - switch packet.(type) { - case *rtcp.PictureLossIndication, *rtcp.FullIntraRequest: - if videoSource != nil { - videoSource.RequestRecoveryFrame() +func logLocalMediaCandidateCounts(report webrtc.StatsReport) { + for _, direction := range []webrtc.StatsType{webrtc.StatsTypeLocalCandidate, webrtc.StatsTypeRemoteCandidate} { + for _, kind := range []webrtc.ICECandidateType{webrtc.ICECandidateTypeHost, webrtc.ICECandidateTypeSrflx, webrtc.ICECandidateTypePrflx, webrtc.ICECandidateTypeRelay} { + udp, tcp := 0, 0 + for _, entry := range report { + candidate, ok := entry.(webrtc.ICECandidateStats) + if !ok || candidate.Type != direction || candidate.CandidateType != kind || candidate.Deleted { + continue + } + switch candidate.Protocol { + case "udp": + udp++ + case "tcp": + tcp++ } } + slog.Debug("screener-client", "event", "media-candidate-counts", "local", true, + "direction", string(direction), "type", kind.String(), "udp", udp, "tcp", tcp) } } } + +// Shutdown must release every transport before waiting on shared source writes. +func closeEdges(edges []*Edge) { + for _, edge := range edges { + _ = edge.connection.Close() + } + for _, edge := range edges { + _ = edge.Close() + } +} diff --git a/internal/client/mediaedge/edge_debug_test.go b/internal/client/mediaedge/edge_debug_test.go new file mode 100644 index 0000000..419fc5c --- /dev/null +++ b/internal/client/mediaedge/edge_debug_test.go @@ -0,0 +1,65 @@ +package mediaedge + +import ( + "bytes" + "encoding/json" + "log/slog" + "strings" + "testing" + + "github.com/pion/webrtc/v4" +) + +func TestLocalMediaCandidateCountsExcludePrivateFields(t *testing.T) { + var output bytes.Buffer + previous := slog.Default() + slog.SetDefault(slog.New(slog.NewJSONHandler(&output, &slog.HandlerOptions{Level: slog.LevelDebug}))) + defer slog.SetDefault(previous) + logLocalMediaCandidateCounts(webrtc.StatsReport{ + "private-local": webrtc.ICECandidateStats{ + ID: "private-local", Type: webrtc.StatsTypeLocalCandidate, + CandidateType: webrtc.ICECandidateTypeHost, Protocol: "udp", + IP: "192.0.2.1", Port: 54321, URL: "private-url", TransportID: "private-transport", + }, + "private-remote": webrtc.ICECandidateStats{ + ID: "private-remote", Type: webrtc.StatsTypeRemoteCandidate, + CandidateType: webrtc.ICECandidateTypeHost, Protocol: "tcp", + }, + "retired": webrtc.ICECandidateStats{ + Type: webrtc.StatsTypeLocalCandidate, CandidateType: webrtc.ICECandidateTypeHost, + Protocol: "udp", Deleted: true, + }, + }) + report := output.String() + for _, private := range []string{"private-", "192.0.2.", "54321"} { + if strings.Contains(report, private) { + t.Fatalf("candidate summary exposed %q", private) + } + } + entries := strings.Split(strings.TrimSpace(report), "\n") + if len(entries) != 8 { + t.Fatalf("got %d candidate count categories, want 8", len(entries)) + } + for _, line := range entries { + var entry struct { + Direction string `json:"direction"` + Type string `json:"type"` + UDP int `json:"udp"` + TCP int `json:"tcp"` + } + if err := json.Unmarshal([]byte(line), &entry); err != nil { + t.Fatal(err) + } + udp, tcp := 0, 0 + if entry.Type == "host" { + if entry.Direction == string(webrtc.StatsTypeLocalCandidate) { + udp = 1 + } else { + tcp = 1 + } + } + if entry.UDP != udp || entry.TCP != tcp { + t.Fatalf("unexpected count: %+v", entry) + } + } +} diff --git a/internal/client/mediaedge/edge_test.go b/internal/client/mediaedge/edge_test.go index fb8dba9..4caeaef 100644 --- a/internal/client/mediaedge/edge_test.go +++ b/internal/client/mediaedge/edge_test.go @@ -1,13 +1,20 @@ package mediaedge import ( + "context" "errors" "net" + "slices" "strings" + "sync" "testing" "time" "github.com/TNTcraftHIM/Screener/internal/client/nativeaudio" + "github.com/TNTcraftHIM/Screener/internal/client/nativecapture" + "github.com/TNTcraftHIM/Screener/internal/media/encoded" + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/streamtracker" "github.com/pion/interceptor" "github.com/pion/logging" "github.com/pion/rtcp" @@ -16,6 +23,204 @@ import ( "github.com/pion/webrtc/v4" ) +func writeSourceFrame(source *Source, data []byte, timestamp, duration time.Duration) error { + if _, err := source.BeginFrame(timestamp); err != nil { + return err + } + return source.WriteVideo(0, encoded.Frame{Data: data, PTS: timestamp, Duration: duration, Recovery: true}) +} + +func TestOutputPlanRetiresOnlyFailedLayerConsumers(t *testing.T) { + engine, err := NewEngine(EngineOptions{BindAddress: "127.0.0.1:0", IncludeLoopback: true}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = engine.Close() }) + wake := make(chan struct{}, 8) + source, err := engine.NewSource("vp8", 2, 2, func() { wake <- struct{}{} }) + if err != nil { + t.Fatal(err) + } + if err = source.ConfigureOutputs([]uint32{150_000, 600_000}); err != nil { + t.Fatal(err) + } + high, _, _ := connectedReceiver(t, engine, source, "high") + low, _, _ := connectedReceiver(t, engine, source, "low") + if _, err = source.BeginFrame(0); err != nil { + t.Fatal(err) + } + for len(wake) > 0 { + <-wake + } + if err = low.SetTargetLayer(0); err != nil { + t.Fatal(err) + } + waitSignal(t, wake, "quiet source target-change wakeup") + plan, err := source.BeginFrame(time.Second / 30) + if err != nil || !slices.Equal(plan.Active, []bool{true, true}) || len(plan.Bitrates) != 2 { + t.Fatalf("output plan = %+v, %v", plan, err) + } + plan.Bitrates[0] = 1 + trackers := source.media.StreamTrackerManager() + lowerTracker, higherTracker := trackers.AddTracker(0), trackers.AddTracker(1) + lowerTracker.Observe(0, 1200, 1188, true, 90000, nil) + higherTracker.Observe(0, 1200, 1188, true, 90000, nil) + if err = source.DisableLayer(0); err != nil { + t.Fatal(err) + } + if low.State() != webrtc.PeerConnectionStateClosed || high.State() != webrtc.PeerConnectionStateConnected { + t.Fatalf("failed/sibling state = %s/%s", low.State(), high.State()) + } + if err = high.SetTargetLayer(0); err == nil { + t.Fatal("failed output accepted a new demand") + } + lowerTracker.Observe(0, 1200, 1188, true, 93000, nil) + if lowerTracker.Status() != streamtracker.StreamStatusStopped || higherTracker.Status() != streamtracker.StreamStatusActive { + t.Fatal("failed tracker revived or reset a healthy sibling") + } + plan, err = source.BeginFrame(2 * time.Second / 30) + if err != nil || !slices.Equal(plan.Active, []bool{false, true}) || plan.Bitrates[0] != 0 || plan.Bitrates[1] != 600_000 { + t.Fatalf("surviving output plan = %+v, %v", plan, err) + } + if err = source.WriteVideo(0, encoded.Frame{Data: sfu.VP8KeyFrame8x8, PTS: 2 * time.Second / 30, Duration: time.Second / 30}); err == nil { + t.Fatal("late output revived a failed encoder slot") + } + for index, input := range []string{"capture", "relay"} { + candidate, _, _ := connectedReceiver(t, engine, source, "failed-demand-"+input) + // Exercise a framework request independently of the explicit Native + // setter. This is admission metadata, not simulated network evidence. + candidate.transport.OnTransportCCFeedback(nil, &rtcp.TransportLayerCC{}) + candidate.transport.Output.SetMaxSpatialLayer(0) + candidate.transport.Output.SetBudget(80_000) + if input == "capture" { + plan, err = source.BeginFrame(time.Duration(index+3) * time.Second / 30) + if err != nil || !slices.Equal(plan.Active, []bool{false, true}) || plan.Bitrates[0] != 0 { + t.Fatalf("failed demand changed the capture plan: %+v, %v", plan, err) + } + } else { + err = source.WriteRTP(&rtp.Packet{Header: rtp.Header{Version: 2, SSRC: 42, PayloadType: 96, + SequenceNumber: 1, Timestamp: 90000, Marker: true}, Payload: append([]byte{0x10}, sfu.VP8KeyFrame8x8...)}) + if err != nil { + t.Fatal(err) + } + } + if candidate.State() != webrtc.PeerConnectionStateClosed || high.State() != webrtc.PeerConnectionStateConnected { + t.Fatalf("%s failed/sibling state = %s/%s", input, candidate.State(), high.State()) + } + } + if err = source.ConfigureOutputs([]uint32{150_000, 600_000}); err != nil { + t.Fatal(err) + } + lowerTracker.Observe(0, 1200, 1188, true, 96000, nil) + if lowerTracker.Status() != streamtracker.StreamStatusActive || high.SetTargetLayer(0) != nil { + t.Fatal("validated capture output did not restore the failed slot") + } +} + +func TestRelayFailureCannotRetireReconfiguredProfile(t *testing.T) { + engine, err := NewEngine(EngineOptions{BindAddress: "127.0.0.1:0", IncludeLoopback: true}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = engine.Close() }) + source, err := engine.NewSource("vp8", 1, 2, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = source.Close() }) + source.relay = &relayDerivation{source: source} + _ = source.SetFormat(1, 640, 360) + profile := nativecapture.VideoProfile{Width: 1280, Height: 720, Framerate: 30, Bitrate: 3_000_000, Preference: "balanced"} + if err = source.SetRelayProfile(profile); err != nil { + t.Fatal(err) + } + low, _, _ := connectedReceiver(t, engine, source, "relay-failure") + if err = low.SetTargetLayer(0); err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithCancelCause(t.Context()) + defer cancel(nil) + done := make(chan struct{}) + close(done) + old := &relayRun{owner: source.relay, ctx: ctx, cancel: cancel, done: done, + plan: relayPlan{profile: source.relayProfile, format: source.formats[1].Load()}} + source.relay.run = old + cancel(errors.New("old decoder failed")) + profile.Preference = "maintain-resolution" + if err = source.SetRelayProfile(profile); err != nil { + t.Fatal(err) + } + profile.Preference = "balanced" + if err = source.SetRelayProfile(profile); err != nil { + t.Fatal(err) + } + if source.relayProfile == old.plan.profile || *source.relayProfile != *old.plan.profile { + t.Fatal("fixture did not return to equal settings with a new owner") + } + retire, err := source.markLayerUnavailable(0, old) + if err != nil || retire != nil || source.outputBitrates[0] == 0 || low.State() != webrtc.PeerConnectionStateConnected { + t.Fatal("late failed decoder retired reconfigured output") + } + current := &relayRun{owner: source.relay, ctx: ctx, cancel: cancel, done: done, + plan: relayPlan{profile: source.relayProfile, format: source.formats[1].Load()}} + source.relay.run = current + retire, err = source.markLayerUnavailable(0, current) + if err != nil || retire == nil || low.State() != webrtc.PeerConnectionStateConnected { + t.Fatal("current failure did not defer transport retirement until after owner locks") + } + retire() + if source.outputBitrates[0] != 0 || low.State() != webrtc.PeerConnectionStateClosed { + t.Fatal("current decoder failure was not retired") + } +} + +func TestShutdownClosesSiblingTransportsBeforeWaitingForWrites(t *testing.T) { + for _, owner := range []string{"source", "engine"} { + t.Run(owner, func(t *testing.T) { + engine, err := NewEngine(EngineOptions{BindAddress: "127.0.0.1:0", IncludeLoopback: true}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = engine.Close() }) + source, err := engine.NewSource("vp8", 2, 1, nil) + if err != nil { + t.Fatal(err) + } + closed := []chan struct{}{make(chan struct{}, 1), make(chan struct{}, 1)} + for index := range closed { + _, err := engine.NewEdge(source, EdgeOptions{ + ConnectionID: "shutdown-" + owner + string(rune('a'+index)), + Events: EdgeEvents{ConnectionState: func(state webrtc.PeerConnectionState, _ *SelectedPair) { + if state == webrtc.PeerConnectionStateClosed { + closed[index] <- struct{}{} + } + }}, + }) + if err != nil { + t.Fatal(err) + } + } + source.writeMu.Lock() + unlock := sync.OnceFunc(source.writeMu.Unlock) + done := make(chan struct{}) + t.Cleanup(func() { unlock(); waitSignal(t, done, "shutdown cleanup") }) + go func() { + if owner == "source" { + _ = source.Close() + } else { + _ = engine.Close() + } + close(done) + }() + for _, transport := range closed { + waitSignal(t, transport, "sibling transport close") + } + unlock() + waitSignal(t, done, "shutdown completion") + }) + } +} + func TestPortMappingCandidateUsesObservedPublicAddress(t *testing.T) { var emitted []*webrtc.ICECandidateInit gathering := &localCandidateGathering{ @@ -58,7 +263,7 @@ func testEncodedSourceFanout(t *testing.T, codec string) { } t.Cleanup(func() { _ = engine.Close() }) keyFrames := make(chan struct{}, 1) - source, err := engine.NewSource(codec, 2, func() { + source, err := engine.NewSource(codec, 2, 1, func() { select { case keyFrames <- struct{}{}: default: @@ -76,7 +281,7 @@ func testEncodedSourceFanout(t *testing.T, codec string) { if edgeA.connection == edgeB.connection || engine.ListenAddress() == "" { t.Fatal("media edges did not keep independent transports") } - if edgeA.bandwidth == edgeB.bandwidth { + if edgeA.transport == edgeB.transport { t.Fatal("media edges shared one bandwidth estimator") } if _, err = engine.NewEdge(source, EdgeOptions{ConnectionID: "edge-c"}); !errors.Is(err, ErrSourceCapacity) { @@ -96,7 +301,7 @@ func testEncodedSourceFanout(t *testing.T, codec string) { if codec == "vp8" { accessUnit = []byte{0x10, 0, 0, 0x9d, 0x01, 0x2a, 0x80, 0x02, 0xe0, 0x01, 0} } - if err = source.WriteVideo(accessUnit, time.Second, time.Second/30); err != nil { + if err = writeSourceFrame(source, accessUnit, time.Second, time.Second/30); err != nil { t.Fatal(err) } first := waitPacket(t, packetA) @@ -108,6 +313,11 @@ func testEncodedSourceFanout(t *testing.T, codec string) { t.Fatalf("VP8 packetizer changed its encoded source: %v", parseErr) } } + // Verify feedback delivery independently of the upstream bootstrap throttle. + source.media.GetAllBuffers()[0].SetPLIThrottle(0) + for len(keyFrames) > 0 { + <-keyFrames + } if err = receiverA.WriteRTCP([]rtcp.Packet{ &rtcp.PictureLossIndication{MediaSSRC: first.SSRC}, }); err != nil { @@ -138,7 +348,7 @@ func TestRemoteCandidatesAreBoundedUntilTheAnswer(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = engine.Close() }) - source, err := engine.NewSource("h264", 1, nil) + source, err := engine.NewSource("h264", 1, 1, nil) if err != nil { t.Fatal(err) } @@ -191,7 +401,7 @@ func TestRepeatedAnswerIsIdempotent(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = engine.Close() }) - source, err := engine.NewSource("h264", 1, nil) + source, err := engine.NewSource("h264", 1, 1, nil) if err != nil { t.Fatal(err) } @@ -215,29 +425,29 @@ func TestCaptureTimestampsDriveTheRTPClock(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = engine.Close() }) - source, err := engine.NewSource("h264", 1, nil) + source, err := engine.NewSource("vp8", 1, 1, nil) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = source.Close() }) _, receiver, packets := connectedReceiver(t, engine, source, "timed-edge") t.Cleanup(func() { _ = receiver.Close() }) - accessUnit := []byte{0, 0, 0, 1, 0x65, 0x88, 0x84, 0x00} + accessUnit := sfu.VP8KeyFrame8x8 start := 10 * time.Second frameDuration := time.Second / 30 - if err = source.WriteVideo(accessUnit, start, frameDuration); err != nil { + if err = writeSourceFrame(source, accessUnit, start, frameDuration); err != nil { t.Fatal(err) } first := waitPacket(t, packets) - if err = source.WriteVideo(accessUnit, start+frameDuration, frameDuration); err != nil { + if err = writeSourceFrame(source, accessUnit, start+frameDuration, frameDuration); err != nil { t.Fatal(err) } second := waitPacket(t, packets) if got := second.Timestamp - first.Timestamp; got < 2_999 || got > 3_001 { t.Fatalf("steady timestamp delta = %d", got) } - if err = source.WriteVideo(accessUnit, start+5*time.Second, frameDuration); err != nil { + if err = writeSourceFrame(source, accessUnit, start+5*time.Second, frameDuration); err != nil { t.Fatal(err) } third := waitPacket(t, packets) @@ -252,20 +462,20 @@ func TestNewCaptureGenerationKeepsTheRTPClockContinuous(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = engine.Close() }) - source, err := engine.NewSource("h264", 1, nil) + source, err := engine.NewSource("vp8", 1, 1, nil) if err != nil { t.Fatal(err) } _, receiver, packets := connectedReceiver(t, engine, source, "generation-edge") t.Cleanup(func() { _ = receiver.Close() }) - accessUnit := []byte{0, 0, 0, 1, 0x65, 0x88, 0x84, 0x00} + accessUnit := sfu.VP8KeyFrame8x8 frameDuration := time.Second / 30 - if err = source.WriteVideo(accessUnit, 50*time.Second, frameDuration); err != nil { + if err = writeSourceFrame(source, accessUnit, 50*time.Second, frameDuration); err != nil { t.Fatal(err) } first := waitPacket(t, packets) source.BeginGeneration() - if err = source.WriteVideo(accessUnit, 100*time.Millisecond, frameDuration); err != nil { + if err = writeSourceFrame(source, accessUnit, 100*time.Millisecond, frameDuration); err != nil { t.Fatal(err) } second := waitPacket(t, packets) @@ -282,7 +492,7 @@ func TestOneLocalBridgeDoesNotConsumeRouteCapacity(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = engine.Close() }) - source, err := engine.NewSource("h264", 1, nil) + source, err := engine.NewSource("h264", 1, 1, nil) if err != nil { t.Fatal(err) } @@ -332,7 +542,7 @@ func TestBandwidthObserverReceivesTransportFeedbackWithoutPacing(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = engine.Close() }) - source, err := engine.NewSource("h264", 1, nil) + source, err := engine.NewSource("h264", 1, 1, nil) if err != nil { t.Fatal(err) } @@ -341,7 +551,7 @@ func TestBandwidthObserverReceivesTransportFeedbackWithoutPacing(t *testing.T) { if !strings.Contains(edge.connection.LocalDescription().SDP, "transport-cc") { t.Fatal("native media offer did not negotiate transport feedback") } - source.SetFormat(1280, 720) + source.SetFormat(0, 1280, 720) started := time.Now() if _, ok := edge.QualitySample(started); ok { t.Fatal("first quality sample did not establish a baseline") @@ -355,7 +565,7 @@ func TestBandwidthObserverReceivesTransportFeedbackWithoutPacing(t *testing.T) { deadline := time.Now().Add(5 * time.Second) captureTimestamp := time.Second for time.Now().Before(deadline) { - if err = source.WriteVideo( + if err = writeSourceFrame(source, accessUnit, captureTimestamp, time.Second/30, @@ -367,7 +577,7 @@ func TestBandwidthObserverReceivesTransportFeedbackWithoutPacing(t *testing.T) { case <-packets: default: } - if bitrate, observed := edge.bandwidth.targetBitrate(); observed && time.Since(started) >= time.Second { + if bitrate, observed := edge.targetBitrate(); observed && time.Since(started) >= time.Second { if bitrate <= 0 { t.Fatalf("observed target bitrate = %d", bitrate) } @@ -390,7 +600,7 @@ func TestAudioUsesTheSamePeerConnectionAndCapacityAsVideo(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = engine.Close() }) - video, err := engine.NewSource("h264", 1, nil) + video, err := engine.NewSource("h264", 1, 1, nil) if err != nil { t.Fatal(err) } @@ -431,7 +641,7 @@ func TestAudioSourceDeliversOpusOnTheVideoPeerConnection(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = engine.Close() }) - video, err := engine.NewSource("h264", 1, nil) + video, err := engine.NewSource("h264", 1, 1, nil) if err != nil { t.Fatal(err) } @@ -483,7 +693,7 @@ func connectedReceiver( } t.Cleanup(func() { _ = edge.Close() }) receiver := newReceiver(t) - packets := make(chan *rtp.Packet, 1) + packets := make(chan *rtp.Packet, 64) receiver.OnTrack(func(track *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { go func() { for { diff --git a/internal/client/mediaedge/engine.go b/internal/client/mediaedge/engine.go index bb44362..62ca4b1 100644 --- a/internal/client/mediaedge/engine.go +++ b/internal/client/mediaedge/engine.go @@ -4,17 +4,20 @@ import ( "context" "errors" "net" + "net/netip" "strconv" "strings" "sync" + "sync/atomic" "time" "github.com/TNTcraftHIM/Screener/internal/client/portmapping" + "github.com/TNTcraftHIM/Screener/internal/media/encoded" + "github.com/TNTcraftHIM/Screener/internal/media/forwarding" "github.com/pion/ice/v4" "github.com/pion/interceptor" "github.com/pion/logging" - "github.com/pion/rtp" - "github.com/pion/rtp/codecs" + "github.com/pion/rtcp" "github.com/pion/stun/v3" "github.com/pion/webrtc/v4" ) @@ -54,18 +57,20 @@ type EngineOptions struct { } type Engine struct { - api *webrtc.API - mux *ice.UniversalUDPMuxDefault - listenAddress string - localPort int - portMapping *portmapping.Mapping - bandwidth *bandwidthObservers - ctx context.Context - cancel context.CancelFunc + api *webrtc.API + settings webrtc.SettingEngine + mux *ice.UniversalUDPMuxDefault + listenAddress string + localPort int + portMapping *portmapping.Mapping + initialBitrate int + ctx context.Context + cancel context.CancelFunc - mu sync.Mutex - edges map[*Edge]struct{} - closed bool + mu sync.Mutex + edges map[*Edge]struct{} + publications map[*Publication]struct{} + closed bool } func NewEngine(options EngineOptions) (*Engine, error) { @@ -82,10 +87,12 @@ func NewEngine(options EngineOptions) (*Engine, error) { return nil, errors.New("native media UDP socket is unavailable") } loggerFactory := logging.NewDefaultLoggerFactory() + ready := make(chan struct{}) mux := ice.NewUniversalUDPMuxDefault(ice.UniversalUDPMuxParams{ Logger: loggerFactory.NewLogger("screener-ice"), - UDPConn: connection, + UDPConn: &initializingUDPConn{UDPConn: connection, ready: ready}, }) + close(ready) settingEngine := webrtc.SettingEngine{LoggerFactory: loggerFactory} settingEngine.SetICEUDPMux(mux) settingEngine.SetIncludeLoopbackCandidate(options.IncludeLoopback) @@ -105,33 +112,29 @@ func NewEngine(options EngineOptions) (*Engine, error) { return nil, err } registry := &interceptor.Registry{} - bandwidth, err := configureBandwidthObservers( - mediaEngine, - registry, - options.InitialBitrate, - ) - if err != nil { - _ = mux.Close() - return nil, err - } if err = webrtc.RegisterDefaultInterceptors(mediaEngine, registry); err != nil { _ = mux.Close() return nil, err } ctx, cancel := context.WithCancel(context.Background()) + if options.InitialBitrate <= 0 { + options.InitialBitrate = defaultNativeSourceInitialBitrate + } engine := &Engine{ api: webrtc.NewAPI( webrtc.WithMediaEngine(mediaEngine), webrtc.WithInterceptorRegistry(registry), webrtc.WithSettingEngine(settingEngine), ), - mux: mux, - listenAddress: connection.LocalAddr().String(), - localPort: connection.LocalAddr().(*net.UDPAddr).Port, - bandwidth: bandwidth, - ctx: ctx, - cancel: cancel, - edges: make(map[*Edge]struct{}), + mux: mux, + settings: settingEngine, + listenAddress: connection.LocalAddr().String(), + localPort: connection.LocalAddr().(*net.UDPAddr).Port, + initialBitrate: options.InitialBitrate, + ctx: ctx, + cancel: cancel, + edges: make(map[*Edge]struct{}), + publications: make(map[*Publication]struct{}), } if options.PortMapping { engine.portMapping = portmapping.Start(connection.LocalAddr().(*net.UDPAddr).Port) @@ -139,6 +142,29 @@ func NewEngine(options EngineOptions) (*Engine, error) { return engine, nil } +// Pion starts its reader before publishing the embedded UDP mux. Both receive +// paths wait for construction; the AddrPort interface preserves its fast path. +type initializingUDPConn struct { + *net.UDPConn + ready <-chan struct{} +} + +var _ ice.AddrPortReaderWriter = (*initializingUDPConn)(nil) + +func (connection *initializingUDPConn) ReadFrom(buffer []byte) (int, net.Addr, error) { + <-connection.ready + return connection.UDPConn.ReadFrom(buffer) +} + +func (connection *initializingUDPConn) ReadFromAddrPort(buffer []byte) (int, netip.AddrPort, error) { + <-connection.ready + return connection.UDPConn.ReadFromUDPAddrPort(buffer) +} + +func (connection *initializingUDPConn) WriteToAddrPort(buffer []byte, address netip.AddrPort) (int, error) { + return connection.UDPConn.WriteToUDPAddrPort(buffer, address) +} + type mappedAddress struct { address string port int @@ -232,47 +258,49 @@ func (engine *Engine) ListenAddress() string { return engine.listenAddress } -func (engine *Engine) NewSource(codec string, capacity int, requestKeyFrame func()) (*Source, error) { - parameters, supported := videoCodecs[codec] +func (engine *Engine) NewSource(codec string, capacity, layers int, requestKeyFrame func()) (*Source, error) { + _, supported := videoCodecs[codec] if !supported { return nil, errors.New("native video codec is unsupported") } if capacity < 1 || capacity > 4 { return nil, errors.New("native media source capacity is outside the route bound") } + if layers < 1 || layers > 3 { + return nil, errors.New("native media source output count is invalid") + } engine.mu.Lock() defer engine.mu.Unlock() if engine.closed { return nil, errors.New("native media engine is closed") } - track, err := webrtc.NewTrackLocalStaticRTP( - parameters.RTPCodecCapability, - "screen", - "screener-native", - ) + source := &Source{ + engine: engine, codec: codec, formats: make([]atomic.Uint64, layers), + capacity: capacity, requestKeyFrame: requestKeyFrame, edges: make(map[*Edge]bool), + publications: make(map[*Publication]struct{}), + } + media, err := forwarding.NewEncodedSource(forwarding.SourceOptions{ + ID: "screen", StreamID: "screener-native", Codec: videoCodecs[codec], + Formats: make([]forwarding.LayerFormat, layers), MaxPackets: encoded.MaxPacketWindow, + OnRTCP: func(layer int, packets []rtcp.Packet) { + for _, packet := range packets { + switch packet.(type) { + case *rtcp.PictureLossIndication, *rtcp.FullIntraRequest: + source.requestLayerKeyFrame(layer) + } + } + }, + }) if err != nil { return nil, err } - var payloader rtp.Payloader = &codecs.H264Payloader{} - if codec == "vp8" { - payloader = &codecs.VP8Payloader{EnablePictureID: true} + source.media = media + if requestKeyFrame != nil { + source.recoveryRequests = make(chan struct{}, 1) + source.recoveryStopped = make(chan struct{}) + go source.runRecovery() } - return &Source{ - engine: engine, - track: track, - codec: codec, - packetizer: rtp.NewPacketizer( - videoPacketMTU, - uint8(parameters.PayloadType), - 0, - payloader, - rtp.NewRandomSequencer(), - videoClockRate, - ), - capacity: capacity, - requestKeyFrame: requestKeyFrame, - edges: make(map[*Edge]bool), - }, nil + return source, nil } func (engine *Engine) register(edge *Edge) error { @@ -303,9 +331,14 @@ func (engine *Engine) Close() error { for edge := range engine.edges { edges = append(edges, edge) } + publications := make([]*Publication, 0, len(engine.publications)) + for publication := range engine.publications { + publications = append(publications, publication) + } engine.mu.Unlock() - for _, edge := range edges { - _ = edge.Close() + closeEdges(edges) + for _, publication := range publications { + _ = publication.Close() } if engine.portMapping != nil { engine.portMapping.Close() diff --git a/internal/client/mediaedge/group_fixture_test.go b/internal/client/mediaedge/group_fixture_test.go new file mode 100644 index 0000000..6b70ac1 --- /dev/null +++ b/internal/client/mediaedge/group_fixture_test.go @@ -0,0 +1,241 @@ +package mediaedge + +import ( + "bytes" + "encoding/hex" + "encoding/json" + "io" + "os" + "testing" + "time" + + "github.com/TNTcraftHIM/Screener/internal/media/encoded" + "github.com/pion/rtp" + "github.com/pion/rtp/codecs" +) + +// Explicit codec gate input; ordinary unit checks do not build/run a native codec. +func TestEncodedGroupVP8Fixture(t *testing.T) { + inputPath := os.Getenv("SCREENER_ENCODED_FIXTURE") + if inputPath == "" { + t.Skip("set SCREENER_ENCODED_FIXTURE to the bounded VP8 fixture JSONL") + } + input, err := os.Open(inputPath) + if err != nil { + t.Fatal(err) + } + defer input.Close() + type fixtureFrame struct { + FrameIndex, Layer, Width, Height int + PTS time.Duration + Recovery bool + DataHex string + } + var rows []fixtureFrame + decoder := json.NewDecoder(input) + decoder.DisallowUnknownFields() + for { + var row fixtureFrame + if err = decoder.Decode(&row); err == io.EOF { + break + } else if err != nil { + t.Fatal(err) + } + if row.FrameIndex < 0 || row.FrameIndex >= 40 || len(rows) >= 96 { + t.Fatal("fixture exceeds the bounded trace") + } + rows = append(rows, row) + } + if len(rows) != 96 { + t.Fatal("fixture must contain 96 access units") + } + engine, err := NewEngine(EngineOptions{BindAddress: "127.0.0.1:0", IncludeLoopback: true}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = engine.Close() }) + source, err := engine.NewSource("vp8", 2, 3, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = source.Close() }) + for layer, dimensions := range [][2]uint32{{160, 90}, {320, 180}, {640, 360}} { + if err = source.SetFormat(layer, dimensions[0], dimensions[1]); err != nil { + t.Fatal(err) + } + } + if err = source.ConfigureOutputs([]uint32{90_000, 300_000, 1_200_000}); err != nil { + t.Fatal(err) + } + a, receiverA, packetsA := connectedReceiver(t, engine, source, "group-a") + b, receiverB, packetsB := connectedReceiver(t, engine, source, "group-b") + t.Cleanup(func() { _ = receiverA.Close() }) + t.Cleanup(func() { _ = receiverB.Close() }) + edges := []*Edge{a, b} + channels := []<-chan *rtp.Packet{packetsA, packetsB} + // Prove both physical outputs before measuring a trace that cannot answer + // asynchronous recovery requests. Repeated real keyframes are warmup only. + warmTicker := time.NewTicker(time.Second / 30) + defer warmTicker.Stop() + var warmFrames [2]uint64 + var partial [2]bool + for frame := 0; frame < 45; frame++ { + pts := time.Duration(frame+1) * time.Second / 30 + if _, err = source.BeginFrame(pts); err != nil { + t.Fatal(err) + } + for _, row := range rows[:3] { + if row.FrameIndex != 0 || !row.Recovery { + t.Fatal("warmup requires the three first-frame recovery outputs") + } + data, decodeErr := hex.DecodeString(row.DataHex) + if decodeErr != nil { + t.Fatal(decodeErr) + } + if err = source.WriteVideo(row.Layer, encoded.Frame{Data: data, PTS: pts, Duration: time.Second / 30, Recovery: true}); err != nil { + t.Fatal(err) + } + } + <-warmTicker.C + for child, channel := range channels { + drain: + for { + select { + case packet := <-channel: + if len(packet.Payload) == 0 { + continue + } + partial[child] = !packet.Marker + if packet.Marker { + warmFrames[child]++ + } + default: + break drain + } + } + } + } + var baselineFrames, baselineBytes [2]uint64 + for child, edge := range edges { + baselineFrames[child], baselineBytes[child], _ = edge.videoCounters() + if warmFrames[child] == 0 || partial[child] || baselineFrames[child] != warmFrames[child] { + t.Fatalf("warmup did not drain child %d: received=%d sent=%d partial=%v", child, warmFrames[child], baselineFrames[child], partial[child]) + } + } + source.BeginGeneration() + targets := [][2]int{{2, 2}, {2, 0}, {0, 0}, {1, 1}, {2, 2}} + type deliveredFrame struct { + Index, Layer, Width, Height int + PTS time.Duration + Recovery bool + Data []byte + } + delivered := [2][]deliveredFrame{} + var previous [2]*rtp.Packet + var previousPTS [2]time.Duration + var payloadBytes [2]uint64 + lastFrame, records := -1, 0 + for _, row := range rows { + records++ + wanted := targets[row.FrameIndex/8] + if row.FrameIndex != lastFrame { + if row.FrameIndex != lastFrame+1 { + t.Fatal("fixture source frames are not contiguous") + } + for child, edge := range edges { + if err = edge.SetTargetLayer(wanted[child]); err != nil { + t.Fatal(err) + } + } + highestNeeded := max(wanted[0], wanted[1]) + for _, edge := range edges { + highestNeeded = max(highestNeeded, int(edge.transport.Output.State().Current)) + } + plan, err := source.BeginFrame(row.PTS) + if err != nil || len(plan.Active) != 3 || !plan.Active[wanted[0]] || !plan.Active[wanted[1]] { + t.Fatalf("requested outputs = %v, active slots = %v, err=%v", wanted, plan.Active, err) + } + for slot, active := range plan.Active { + if active && slot > highestNeeded { + t.Fatalf("slot %d is active above current/requested output %d", slot, highestNeeded) + } + } + lastFrame = row.FrameIndex + } + data, err := hex.DecodeString(row.DataHex) + if err != nil { + t.Fatal(err) + } + if err = source.WriteVideo(row.Layer, encoded.Frame{ + Data: data, PTS: row.PTS, Duration: time.Second / 30, Recovery: row.Recovery, + }); err != nil { + t.Fatal(err) + } + for child, layer := range wanted { + if row.Layer != layer { + continue + } + var received []byte + var firstTS uint32 + for { + packet := waitPacket(t, channels[child]) + payloadBytes[child] += uint64(len(packet.Payload)) + if previous[child] != nil && packet.SequenceNumber != previous[child].SequenceNumber+1 { + t.Fatal("layer switch interrupted outbound RTP sequence") + } + if len(received) == 0 { + firstTS = packet.Timestamp + if previous[child] != nil { + delta := packet.Timestamp - previous[child].Timestamp + want := (row.PTS - previousPTS[child]).Seconds() * videoClockRate + if float64(delta) < want-1 || float64(delta) > want+1 { + t.Fatal("layer switch changed source time mapping") + } + } + } else if packet.Timestamp != firstTS { + t.Fatal("access unit mixed two RTP timestamps") + } + payload, parseErr := (&codecs.VP8Packet{}).Unmarshal(packet.Payload) + if parseErr != nil { + t.Fatal(parseErr) + } + received = append(received, payload...) + previous[child] = packet + if packet.Marker { + break + } + } + if !bytes.Equal(received, data) { + t.Fatalf("source frame %d child %d selected layer %d: received %d bytes, expected %d; state=%+v", + row.FrameIndex, child, row.Layer, len(received), len(data), edges[child].transport.Output.State()) + } + previousPTS[child] = row.PTS + delivered[child] = append(delivered[child], deliveredFrame{ + Index: row.FrameIndex, Layer: row.Layer, Width: row.Width, Height: row.Height, + PTS: row.PTS, Recovery: row.Recovery, Data: received, + }) + } + } + if records != 96 || len(delivered[0]) != 40 || len(delivered[1]) != 40 { + t.Fatalf("input=%d delivered=%d,%d", records, len(delivered[0]), len(delivered[1])) + } + for child, edge := range edges { + frames, bytes, _ := edge.videoCounters() + if frames-baselineFrames[child] != 40 { + t.Fatal("sender evidence counted other representations") + } + if bytes-baselineBytes[child] != payloadBytes[child] { + t.Fatal("sender evidence counted unused-layer bytes") + } + } + if outputPath := os.Getenv("SCREENER_ENCODED_OUTPUT"); outputPath != "" { + data, err := json.Marshal(delivered) + if err != nil { + t.Fatal(err) + } + if err = os.WriteFile(outputPath, append(data, '\n'), 0600); err != nil { + t.Fatal(err) + } + } + t.Log("96 real VP8 outputs; two unchanged connections delivered 40 selected frames each") +} diff --git a/internal/client/mediaedge/groups.go b/internal/client/mediaedge/groups.go new file mode 100644 index 0000000..15d1a5c --- /dev/null +++ b/internal/client/mediaedge/groups.go @@ -0,0 +1,252 @@ +package mediaedge + +import ( + "errors" + "time" + + "github.com/TNTcraftHIM/Screener/internal/media/encoded" + "github.com/TNTcraftHIM/Screener/internal/media/forwarding" + "github.com/pion/rtcp" + "github.com/pion/webrtc/v4" +) + +// Transport owns current attachment; this map owns only requested membership. +// An independent keyframe installs a requested source without replacing its PC. +type groupConsumer interface { + CurrentSource() *forwarding.Source + ReplaceSource(*forwarding.Source) error +} + +type outputGroup struct { + slot int + media *forwarding.EncodedSource + budget uint32 + active bool + start time.Duration +} + +type outputDemand struct { + consumer groupConsumer + budget uint32 + active bool + lower bool + original bool +} + +func (source *Source) groupForMedia(media *forwarding.Source) *outputGroup { + for _, group := range source.groups { + if group.media.Source == media { + return group + } + } + return nil +} + +func (source *Source) collectDemands(edges []*Edge, publications []*Publication) []outputDemand { + demands := make([]outputDemand, 0, len(edges)+len(publications)) + ceiling := source.media.TrackInfo().Layers[0].Bitrate + for _, edge := range edges { + state := edge.transport.Output.State() + _, observed := edge.transport.TargetBitrate() + active := edge.State() == webrtc.PeerConnectionStateConnected && state.VideoBudget > 0 + lower := active && !edge.local && observed && + (state.Target == 0 || state.Current == 0 && !state.Paused || state.Paused && state.Prepare == 0) + demands = append(demands, outputDemand{consumer: edge.transport, active: active, lower: lower, + original: edge.transport.RequiredActiveCount() > 1, + budget: uint32(max(1000, min(state.VideoBudget, int64(ceiling))))}) + } + for _, publication := range publications { + required := publication.RequiredActiveCount() + active := required > 0 + budget := publication.LowestLayerBudget() + if budget <= 0 { + budget = int64(ceiling) + } + demands = append(demands, outputDemand{consumer: publication.transport, active: active, lower: active, + original: required > 1, + budget: uint32(max(1000, min(budget, int64(ceiling))))}) + } + return demands +} + +// Equal connection-level requests share one correction/adaptation owner. The +// group's RTP-to-codec conversion is applied once, not used as a membership key. +func (source *Source) planGroups(demands []outputDemand) (OutputPlan, error) { + plan := OutputPlan{Active: make([]bool, len(source.outputBitrates)), + Bitrates: append([]uint32(nil), source.outputBitrates...)} + if len(source.formats) != 2 || len(plan.Bitrates) < 2 { + return plan, errors.New("shared output groups need an original and derived slot") + } + if source.groups == nil { + source.groups = map[int]*outputGroup{0: {slot: 0, media: source.media, budget: source.outputBitrates[0]}} + source.memberships = make(map[groupConsumer]*outputGroup) + } + present := make(map[groupConsumer]bool, len(demands)) + hasLowerDemand := false + for _, demand := range demands { + present[demand.consumer] = true + hasLowerDemand = hasLowerDemand || demand.active && demand.lower + plan.Active[1] = plan.Active[1] || demand.active && demand.original + } + for consumer := range source.memberships { + if !present[consumer] { + delete(source.memberships, consumer) + } + } + for _, demand := range demands { + if !demand.active || !demand.lower { + if source.memberships[demand.consumer] != nil { + source.groupRecovery.Or(2) + } + source.memberships[demand.consumer] = nil + continue + } + var selected *outputGroup + for slot := 0; slot < len(source.outputBitrates); slot++ { + group := source.groups[slot] + if group != nil && source.outputBitrates[slot] > 0 && group.budget == demand.budget { + selected = group + break + } + } + if selected == nil { + // A group's rates can change in place only when no other current or + // requested member still needs a different budget. + ordered := []*outputGroup{source.memberships[demand.consumer], source.groupForMedia(demand.consumer.CurrentSource())} + for slot := 0; slot < len(source.outputBitrates); slot++ { + if group := source.groups[slot]; group != nil { + ordered = append(ordered, group) + } + } + for _, group := range ordered { + if group == nil { + continue + } + if source.outputBitrates[group.slot] == 0 { + continue + } + compatible := true + for _, other := range demands { + if other.active && other.lower && other.budget != demand.budget && + (other.consumer.CurrentSource() == group.media.Source || source.memberships[other.consumer] == group) { + compatible = false + break + } + } + if compatible { + selected = group + break + } + } + } + if selected == nil { + for slot := 2; slot < len(source.outputBitrates); slot++ { + if source.groups[slot] == nil && source.outputBitrates[slot] > 0 { + var err error + selected, err = source.newGroup(slot) + if err != nil { + return plan, err + } + break + } + } + } + if selected == nil { + // Existing delivery remains while a prepared split releases its old + // membership. No timer, retry queue or temporary capacity exemption. + continue + } + selected.budget = demand.budget + if !selected.active { + selected.start = source.inputPTS + } + if source.memberships[demand.consumer] != selected && demand.consumer.CurrentSource() != selected.media.Source { + source.groupRecovery.Or(uint32(1) << selected.slot) + } + source.memberships[demand.consumer] = selected + } + for _, group := range source.groups { + active := group.slot == 0 && plan.Active[1] && !hasLowerDemand + for _, demand := range demands { + if !demand.active { + continue + } + if source.memberships[demand.consumer] == group || + demand.consumer.CurrentSource() == group.media.Source && (demand.lower || !demand.original || group.slot != 0) { + active = true + } + } + group.active = active && source.outputBitrates[group.slot] > 0 + plan.Active[group.slot] = group.active + if group.active { + plan.Bitrates[group.slot] = uint32(max(1000, min(group.media.CodecBudget(0, int64(group.budget)), int64(source.outputBitrates[group.slot])))) + } + } + return plan, nil +} + +func (source *Source) newGroup(slot int) (*outputGroup, error) { + info := source.media.TrackInfo() + formats := make([]forwarding.LayerFormat, len(info.Layers)) + for layer, format := range info.Layers { + formats[layer] = forwarding.LayerFormat{Width: format.Width, Height: format.Height, Bitrate: format.Bitrate} + } + formats[0].Bitrate = source.outputBitrates[slot] + media, err := forwarding.NewEncodedSource(forwarding.SourceOptions{ + ID: string(source.media.TrackID()), StreamID: source.media.StreamID(), Codec: videoCodecs[source.codec], + Formats: formats, MaxPackets: encoded.MaxPacketWindow, + OnRTCP: func(layer int, packets []rtcp.Packet) { + for _, packet := range packets { + switch packet.(type) { + case *rtcp.PictureLossIndication, *rtcp.FullIntraRequest: + if layer == 0 { + source.groupRecovery.Or(uint32(1) << slot) + } else { + source.groupRecovery.Or(2) + } + source.RequestRecoveryFrame() + } + } + }, + }) + if err != nil { + return nil, err + } + media.SetLowestLayerRateControlled(true) + group := &outputGroup{slot: slot, media: media, start: source.inputPTS} + if source.hasInput { + if err = media.BeginFrame(source.inputPTS, source.inputAt); err != nil { + media.Close() + return nil, err + } + } + source.groups[slot] = group + return group, nil +} + +func (source *Source) installGroup(group *outputGroup, original bool) { + next := source.media.Source + if group != nil { + next = group.media.Source + } + for consumer, wanted := range source.memberships { + if wanted == group && (original || group != nil) && consumer.CurrentSource() != next { + _ = consumer.ReplaceSource(next) + } + } +} + +func (source *Source) beginGroupFrame(pts time.Duration, at time.Time) error { + if err := source.media.BeginFrame(pts, at); err != nil { + return err + } + source.inputPTS, source.inputAt, source.hasInput = pts, at, true + for slot, group := range source.groups { + if slot != 0 { + if err := group.media.BeginFrame(pts, at); err != nil { + return err + } + } + } + return nil +} diff --git a/internal/client/mediaedge/groups_test.go b/internal/client/mediaedge/groups_test.go new file mode 100644 index 0000000..25f4c64 --- /dev/null +++ b/internal/client/mediaedge/groups_test.go @@ -0,0 +1,364 @@ +package mediaedge + +import ( + "testing" + "time" + + "github.com/TNTcraftHIM/Screener/internal/media/encoded" + "github.com/TNTcraftHIM/Screener/internal/media/forwarding" + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/pion/rtcp" +) + +type groupTestConsumer struct{ source *forwarding.Source } + +func (consumer *groupTestConsumer) CurrentSource() *forwarding.Source { return consumer.source } +func (consumer *groupTestConsumer) ReplaceSource(next *forwarding.Source) error { + consumer.source = next + return nil +} + +func TestOutputGroupsShareSplitRejoinAndRetire(t *testing.T) { + engine := &Engine{} + source, err := engine.NewSource("vp8", 2, 2, nil) + if err != nil { + t.Fatal(err) + } + defer source.Close() + if err = source.SetFormat(0, 640, 360); err != nil { + t.Fatal(err) + } + if err = source.SetFormat(1, 1280, 720); err != nil { + t.Fatal(err) + } + if err = source.ConfigureOutputs([]uint32{500_000, 2_000_000, 500_000, 500_000}); err != nil { + t.Fatal(err) + } + a, b := &groupTestConsumer{source.media.Source}, &groupTestConsumer{source.media.Source} + demands := []outputDemand{{consumer: a, budget: 100_000, active: true, lower: true}, {consumer: b, budget: 100_000, active: true, lower: true}} + var tick time.Duration + plan := func() OutputPlan { + t.Helper() + source.writeMu.Lock() + defer source.writeMu.Unlock() + tick += time.Second + if err := source.beginGroupFrame(tick, time.Unix(0, 0).Add(tick)); err != nil { + t.Fatal(err) + } + value, err := source.planGroups(demands) + if err != nil { + t.Fatal(err) + } + return value + } + shared := plan() + if source.memberships[a] != source.memberships[b] || len(source.groups) != 1 || shared.Bitrates[0] != 100_000 || shared.Active[1] { + t.Fatal("compatible consumers did not share one output") + } + demands[1].budget = 300_000 + split := plan() + ga, gb := source.memberships[a], source.memberships[b] + if ga == gb || split.Bitrates[ga.slot] != 100_000 || split.Bitrates[gb.slot] != 300_000 || b.CurrentSource() != source.media.Source { + t.Fatal("different budgets coupled or moved before recovery") + } + source.installGroup(gb, false) + if b.CurrentSource() != gb.media.Source { + t.Fatal("recovery did not install prepared group") + } + demands[0].budget = 60_000 + changed := plan() + if source.memberships[a] != ga || source.memberships[b] != gb || changed.Bitrates[gb.slot] != 300_000 { + t.Fatal("one consumer's rate change altered its sibling") + } + demands[0].budget = 300_000 + plan() + if source.memberships[a] != gb || a.CurrentSource() != source.media.Source { + t.Fatal("compatible rejoin bypassed recovery") + } + source.installGroup(gb, false) + rejoined := plan() + if rejoined.Active[ga.slot] || !rejoined.Active[gb.slot] { + t.Fatal("unused encoder did not retire after rejoin") + } + demands[0].lower = false + demands[0].original = true + pending := plan() + if !pending.Active[1] || !pending.Active[gb.slot] || a.CurrentSource() != gb.media.Source { + t.Fatal("original handoff retired active source") + } + source.installGroup(nil, true) + if a.CurrentSource() != source.media.Source || b.CurrentSource() != gb.media.Source { + t.Fatal("original handoff moved sibling") + } + demands = demands[:1] + retired := plan() + if retired.Active[gb.slot] { + t.Fatal("last lower member did not release its encoder") + } + demands[0].lower = true + demands[0].original = false + demands[0].budget = 300_000 + reactivated := plan() + if !reactivated.Active[gb.slot] || reactivated.Active[1] || source.memberships[a] != gb { + t.Fatal("inactive compatible group could not reactivate") + } + if err = gb.media.BeginFrame(tick+time.Nanosecond, time.Unix(0, 0).Add(tick+time.Nanosecond)); err != nil { + t.Fatal(err) + } +} + +type groupLowOnlyReceiver struct{ sfu.TrackReceiver } + +func (receiver groupLowOnlyReceiver) GetLayeredBitrate() ([]int32, sfu.Bitrates) { + return []int32{0}, sfu.Bitrates{{100_000}} +} + +func TestOutputGroupsHonorOriginalTransportDemand(t *testing.T) { + check := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + engine, err := NewEngine(EngineOptions{BindAddress: "127.0.0.1:0", IncludeLoopback: true}) + check(err) + defer engine.Close() + source, err := engine.NewSource("vp8", 2, 2, nil) + check(err) + defer source.Close() + check(source.SetFormat(0, 640, 360)) + check(source.SetFormat(1, 1280, 720)) + check(source.ConfigureOutputs([]uint32{500_000, 2_000_000, 500_000})) + edge, receiver, _ := connectedReceiver(t, engine, source, "original-demand") + defer receiver.Close() + plan := func(label string, original bool) { + t.Helper() + source.writeMu.Lock() + defer source.writeMu.Unlock() + value, err := source.planGroups(source.collectDemands([]*Edge{edge}, nil)) + check(err) + if !value.Active[0] || value.Active[1] != original { + t.Fatalf("%s plan = %+v, state = %+v", label, value, edge.transport.Output.State()) + } + } + plan("initial acquisition", true) + output := edge.transport.Output + deadline := time.Now().Add(3 * time.Second) + for frame := 0; output.State().Current != 1; frame++ { + if time.Now().After(deadline) { + t.Fatalf("original never forwarded: %+v", output.State()) + } + pts := time.Duration(frame) * time.Second / 30 + _, err = source.BeginFrame(pts) + check(err) + for layer := range 2 { + check(source.WriteVideo(layer, encoded.Frame{Data: sfu.VP8KeyFrame8x8, PTS: pts, + Duration: time.Second / 30, Recovery: true})) + } + time.Sleep(time.Second / 30) + } + plan("original target", true) + check(edge.SetTargetLayer(0)) + if state := output.State(); state.Target != 0 || state.Current != 1 || state.Paused { + t.Fatalf("fixture did not retain current original during lower handoff: %+v", state) + } + plan("original still current", true) + output.BeginFrame() + plan("lower handoff", false) + edge.local = true + plan("local lower preview", false) + edge.local = false + + // Only the low representation is available; metadata may prepare the + // dormant original, but may not invent its measured availability. + output.SetReceiver(groupLowOnlyReceiver{TrackReceiver: output.Receiver()}) + edge.transport.OnTransportCCFeedback(nil, &rtcp.TransportLayerCC{}) + output.SetMaxSpatialLayer(1) + output.SetBudget(200_000) + plan("limited budget", false) + output.SetBudget(2_000_000) + if state := output.State(); state.Target != 0 || state.Prepare != 1 { + t.Fatalf("higher budget did not prepare the dormant original: %+v", state) + } + plan("higher budget preparation", true) + + publication, err := engine.NewPublication(source, EdgeOptions{ConnectionID: "original-publication"}) + check(err) + check(publication.transport.SetConnected()) + for _, count := range []int{1, 2, 0} { + check(publication.transport.SetActiveCount(count)) + source.writeMu.Lock() + value, err := source.planGroups(source.collectDemands(nil, []*Publication{publication})) + source.writeMu.Unlock() + check(err) + if value.Active[0] != (count > 0) || value.Active[1] != (count > 1) { + t.Fatalf("publication demand %d plan = %+v", count, value) + } + } +} + +func newGroupLifecycleFixture(t *testing.T) (*Source, *groupTestConsumer, *outputGroup, []outputDemand) { + t.Helper() + source, err := (&Engine{}).NewSource("vp8", 2, 2, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = source.Close() }) + for layer, size := range [][2]uint32{{640, 360}, {1280, 720}} { + if err = source.SetFormat(layer, size[0], size[1]); err != nil { + t.Fatal(err) + } + } + if err = source.ConfigureOutputs([]uint32{500_000, 2_000_000, 500_000, 500_000}); err != nil { + t.Fatal(err) + } + a, b := &groupTestConsumer{source.media.Source}, &groupTestConsumer{source.media.Source} + demands := []outputDemand{{consumer: a, budget: 100_000, active: true, lower: true}, + {consumer: b, budget: 300_000, active: true, lower: true}} + if err = source.beginGroupFrame(10*time.Second, time.Unix(100, 0)); err != nil { + t.Fatal(err) + } + if _, err = source.planGroups(demands); err != nil { + t.Fatal(err) + } + group := source.memberships[b] + if group == nil || group.slot < 2 { + t.Fatal("fixture did not prepare an independent lower group") + } + return source, b, group, demands +} + +func TestOutputGroupRejectsRecoveryBeforeChangingAttachment(t *testing.T) { + for _, path := range []string{"derived", "original"} { + for _, stamp := range []string{"future", "duplicate", "before-generation-start"} { + t.Run(path+"/"+stamp, func(t *testing.T) { + source, consumer, group, _ := newGroupLifecycleFixture(t) + slot := group.slot + if path == "original" { + slot = 1 + } + frame := encoded.Frame{Data: sfu.VP8KeyFrame8x8, PTS: 10 * time.Second, + Duration: time.Second / 30, Recovery: true} + if err := source.WriteVideo(slot, frame); err != nil { + t.Fatal(err) + } + if path == "original" { + source.installGroup(group, false) + source.memberships[consumer] = nil + } else { + consumer.source = source.media.Source + } + previous := consumer.CurrentSource() + if stamp == "future" { + frame.PTS = 11 * time.Second + } else if stamp == "before-generation-start" { + source.BeginGeneration() + if err := source.beginGroupFrame(time.Second, time.Unix(200, 0)); err != nil { + t.Fatal(err) + } + frame.PTS = time.Second / 2 + } + if err := source.WriteVideo(slot, frame); err == nil { + t.Fatal("recovery without a valid input timestamp was accepted") + } + if consumer.CurrentSource() != previous { + t.Fatal("rejected recovery changed the consumer's source") + } + }) + } + } +} + +func TestOutputGroupReactivationRejectsEarlierAccessUnits(t *testing.T) { + source, consumer, group, demands := newGroupLifecycleFixture(t) + source.installGroup(group, false) + demands[1].lower = false + demands[1].original = true + if _, err := source.planGroups(demands); err != nil { + t.Fatal(err) + } + source.installGroup(nil, true) + if _, err := source.planGroups(demands); err != nil { + t.Fatal(err) + } + if group.active { + t.Fatal("retired group remained active") + } + if err := source.beginGroupFrame(12*time.Second, time.Unix(102, 0)); err != nil { + t.Fatal(err) + } + demands[1].lower = true + demands[1].original = false + if _, err := source.planGroups(demands); err != nil { + t.Fatal(err) + } + frame := encoded.Frame{Data: sfu.VP8KeyFrame8x8, PTS: 11 * time.Second, + Duration: time.Second / 30, Recovery: true} + if err := source.WriteVideo(group.slot, frame); err != nil { + t.Fatal(err) + } + if consumer.CurrentSource() != source.media.Source { + t.Fatal("earlier recovery installed a reactivated group") + } + if err := source.WriteVideo(1, frame); err != nil { + t.Fatal(err) + } + if report := group.media.GetAllBuffers()[1].GetSenderReportData(); report != nil { + t.Fatal("reactivated group borrowed an earlier original access unit") + } + frame.PTS = 12 * time.Second + if err := source.WriteVideo(group.slot, frame); err != nil { + t.Fatal(err) + } + if consumer.CurrentSource() != group.media.Source { + t.Fatal("current recovery did not install the reactivated group") + } +} + +func TestOutputGroupGenerationRefreshesMetadataAndClock(t *testing.T) { + source, consumer, group, _ := newGroupLifecycleFixture(t) + frame := encoded.Frame{Data: sfu.VP8KeyFrame8x8, PTS: 10 * time.Second, + Duration: time.Second / 30, Recovery: true} + if err := source.WriteVideo(group.slot, frame); err != nil { + t.Fatal(err) + } + previous := group.media.GetAllBuffers()[0].GetSenderReportData() + if previous == nil || consumer.CurrentSource() != group.media.Source { + t.Fatal("group did not establish its original clock and attachment") + } + previousTimestamp := previous.RtpTimestamp + source.BeginGeneration() + for layer, size := range [][2]uint32{{426, 240}, {854, 480}} { + if err := source.SetFormat(layer, size[0], size[1]); err != nil { + t.Fatal(err) + } + } + if err := source.ConfigureOutputs([]uint32{400_000, 1_600_000, 400_000, 400_000}); err != nil { + t.Fatal(err) + } + if err := source.SetFormat(group.slot, 320, 180); err != nil { + t.Fatal(err) + } + info := group.media.TrackInfo() + if source.groups[group.slot] != group || consumer.CurrentSource() != group.media.Source || + info.Layers[0].Width != 320 || info.Layers[0].Height != 180 || info.Layers[0].Bitrate != 400_000 || + info.Layers[1].Width != 854 || info.Layers[1].Height != 480 || info.Layers[1].Bitrate != 1_600_000 { + t.Fatal("replacement source changed group identity or retained old metadata") + } + at := time.Unix(200, 0) + if err := source.beginGroupFrame(time.Second, at); err != nil { + t.Fatal(err) + } + frame.PTS = time.Second + if err := source.WriteVideo(group.slot, frame); err != nil { + t.Fatal(err) + } + report := group.media.GetAllBuffers()[0].GetSenderReportData() + if report == nil || report.RtpTimestamp-previousTimestamp < 2999 || report.RtpTimestamp-previousTimestamp > 3001 { + t.Fatal("new capture generation broke the group RTP clock") + } + correlated, ok := group.media.ReferenceTime(0, report.RtpTimestamp) + if !ok || correlated.Sub(at) < -time.Microsecond || correlated.Sub(at) > time.Microsecond { + t.Fatalf("new generation correlation = %v, available=%v", correlated, ok) + } +} diff --git a/internal/client/mediaedge/publication.go b/internal/client/mediaedge/publication.go new file mode 100644 index 0000000..b1c4b6b --- /dev/null +++ b/internal/client/mediaedge/publication.go @@ -0,0 +1,228 @@ +package mediaedge + +import ( + "errors" + "slices" + "sync" + "time" + + "github.com/TNTcraftHIM/Screener/internal/media/forwarding" + "github.com/pion/webrtc/v4" +) + +type PublicationMedia struct { + Codec string `json:"codec"` + Layers []PublicationLayer `json:"layers"` + Audio bool `json:"audio"` + AudioBitrate uint32 `json:"audioBitrate"` +} + +type PublicationLayer struct { + RID string `json:"rid"` + Width uint32 `json:"width"` + Height uint32 `json:"height"` + Bitrate uint32 `json:"bitrate"` +} + +type Publication struct { + source *Source + audio *AudioSource + transport *forwarding.Publication + // Reuse the existing bounded SDP/candidate owner without registering a P2P edge. + signaling Edge + closeOnce sync.Once + closeErr error + qualityMu sync.Mutex + qualityAt time.Time + qualityCounters forwarding.PublicationCounters +} + +func (engine *Engine) NewPublication(source *Source, options EdgeOptions) (*Publication, error) { + if source == nil || source.engine != engine || options.Local || options.ConnectionID == "" || + len(options.ConnectionID) > 256 || options.Audio != nil && options.Audio.engine != engine { + return nil, errors.New("native publication input is invalid") + } + source.mu.Lock() + unavailable := slices.Contains(source.outputBitrates, uint32(0)) + source.mu.Unlock() + if unavailable { + return nil, errors.New("native publication output is unavailable") + } + if err := source.reserve(false); err != nil { + return nil, err + } + if options.Audio != nil { + if err := options.Audio.reserve(false); err != nil { + source.releaseReservation(false) + return nil, err + } + } + publication := &Publication{source: source, audio: options.Audio} + var audio webrtc.TrackLocal + if options.Audio != nil { + audio = options.Audio.track + } + transport, err := forwarding.NewPublication(forwarding.TransportOptions{ + Source: source.media.Source, Settings: engine.settings, ConnectionID: options.ConnectionID, + InitialBitrate: engine.initialBitrate, Audio: audio, + }) + if err != nil { + _ = publication.Close() + return nil, err + } + publication.transport = transport + transport.SetAudioBitrate(options.Audio.configuredBitrate()) + publication.signaling.connection = transport.PC + publication.signaling.localCandidates = newLocalCandidateGathering(engine, options.ICEServers, 0, options.Events.LocalCandidate) + transport.PC.OnICECandidate(publication.signaling.localCandidates.addPion) + transport.PC.OnConnectionStateChange(func(state webrtc.PeerConnectionState) { + if state == webrtc.PeerConnectionStateConnected { + if connectErr := transport.SetConnected(); connectErr != nil { + _ = transport.PC.Close() + return + } + source.RequestRecoveryFrame() + } + if options.Events.ConnectionState != nil { + options.Events.ConnectionState(state, nil) + } + }) + engine.mu.Lock() + source.mu.Lock() + if engine.closed || source.closed || len(source.publications) >= 2 || slices.Contains(source.outputBitrates, uint32(0)) { + source.mu.Unlock() + engine.mu.Unlock() + _ = publication.Close() + return nil, errors.New("native publication is unavailable") + } + source.publications[publication] = struct{}{} + engine.publications[publication] = struct{}{} + source.mu.Unlock() + engine.mu.Unlock() + return publication, nil +} + +func (publication *Publication) CreateOffer() (webrtc.SessionDescription, error) { + return publication.signaling.CreateOffer() +} +func (publication *Publication) SetAnswer(answer webrtc.SessionDescription) error { + return publication.signaling.SetAnswer(answer) +} +func (publication *Publication) AddRemoteCandidate(candidate *webrtc.ICECandidateInit) error { + return publication.signaling.AddRemoteCandidate(candidate) +} +func (publication *Publication) SetActiveCount(count int) error { + if err := publication.transport.SetActiveCount(count); err != nil { + return err + } + publication.source.RequestRecoveryFrame() + return nil +} +func (publication *Publication) RequiredActiveCount() int { + return publication.transport.RequiredActiveCount() +} +func (publication *Publication) LowestLayerBudget() int64 { + return publication.transport.LowestLayerBudget() +} + +type PublicationQualitySample struct { + SampleTimestampMs int64 `json:"sampleTimestampMs"` + SampleWindowMs int64 `json:"sampleWindowMs"` + RTPStatsID string `json:"rtpStatsId"` + TrackIdentifier string `json:"trackIdentifier"` + Codec string `json:"codec"` + VideoEncodingCount int `json:"videoEncodingCount"` + ActiveVideoEncodingCount int `json:"activeVideoEncodingCount"` + RID string `json:"rid"` + IntervalFramesSent uint64 `json:"intervalFramesSent"` + FramesPerSecond float64 `json:"framesPerSecond"` + Width uint32 `json:"width"` + Height uint32 `json:"height"` + BitrateKbps float64 `json:"bitrateKbps"` + AudioBitrateKbps float64 `json:"audioBitrateKbps"` + AvailableOutgoingKbps *float64 `json:"availableOutgoingKbps"` + State string `json:"state"` + Reason *string `json:"reason"` +} + +func (publication *Publication) QualitySample(now time.Time) (PublicationQualitySample, bool) { + current := publication.transport.Counters() + publication.qualityMu.Lock() + previous, at := publication.qualityCounters, publication.qualityAt + publication.qualityCounters, publication.qualityAt = current, now + publication.qualityMu.Unlock() + window := now.Sub(at) + if at.IsZero() || window < time.Second || window > 5*time.Second || + current.VideoBytes < previous.VideoBytes || current.AudioBytes < previous.AudioBytes { + return PublicationQualitySample{}, false + } + seconds := window.Seconds() + source := publication.transport.CurrentSource() + sample := PublicationQualitySample{SampleTimestampMs: now.UnixMilli(), SampleWindowMs: window.Milliseconds(), + RTPStatsID: publication.signaling.connection.ID(), TrackIdentifier: string(source.TrackID()), + Codec: publication.source.codec, VideoEncodingCount: len(current.Frames), State: "unknown", + BitrateKbps: float64((current.VideoBytes-previous.VideoBytes+current.AudioBytes-previous.AudioBytes)*8) / seconds / 1000, + AudioBitrateKbps: float64((current.AudioBytes-previous.AudioBytes)*8) / seconds / 1000} + for index, frames := range current.Frames { + if index >= len(previous.Frames) || frames <= previous.Frames[index] { + continue + } + sample.ActiveVideoEncodingCount++ + sample.IntervalFramesSent = frames - previous.Frames[index] + sample.RID = forwarding.PublicationRID(index, len(current.Frames)) + sample.Width, sample.Height = sentVideoDimensions(source, index) + } + sample.FramesPerSecond = float64(sample.IntervalFramesSent) / seconds + if current.Observed { + capacity := float64(current.TargetBitrate) / 1000 + sample.AvailableOutgoingKbps = &capacity + if publication.signaling.State() == webrtc.PeerConnectionStateConnected && sample.IntervalFramesSent > 0 { + reason := "none" + sample.State = "healthy" + if current.Limited { + reason, sample.State = "bandwidth", "degraded" + } + sample.Reason = &reason + } + } + return sample, true +} + +func (publication *Publication) Media() PublicationMedia { + info := publication.transport.CurrentSource().TrackInfo() + media := PublicationMedia{Codec: publication.source.codec, Audio: publication.audio != nil, + AudioBitrate: publication.audio.configuredBitrate(), Layers: make([]PublicationLayer, len(info.Layers))} + for index, layer := range info.Layers { + media.Layers[index] = PublicationLayer{RID: forwarding.PublicationRID(index, len(info.Layers)), + Width: layer.Width, Height: layer.Height, Bitrate: layer.Bitrate} + } + publication.transport.SetAudioBitrate(media.AudioBitrate) + return media +} + +func (publication *Publication) Close() error { + publication.closeOnce.Do(func() { + publication.signaling.mu.Lock() + publication.signaling.closed = true + if publication.signaling.localCandidates != nil { + publication.signaling.localCandidates.close() + } + publication.signaling.mu.Unlock() + if publication.transport != nil { + publication.closeErr = publication.transport.Close() + } + publication.source.mu.Lock() + delete(publication.source.publications, publication) + publication.source.mu.Unlock() + publication.source.releaseReservation(false) + if publication.audio != nil { + publication.audio.releaseReservation(false) + } + engine := publication.source.engine + engine.mu.Lock() + delete(engine.publications, publication) + engine.mu.Unlock() + publication.source.RequestRecoveryFrame() + }) + return publication.closeErr +} diff --git a/internal/client/mediaedge/publication_test.go b/internal/client/mediaedge/publication_test.go new file mode 100644 index 0000000..3de25c7 --- /dev/null +++ b/internal/client/mediaedge/publication_test.go @@ -0,0 +1,178 @@ +package mediaedge + +import ( + "errors" + "testing" + "time" + + "github.com/TNTcraftHIM/Screener/internal/media/encoded" + "github.com/TNTcraftHIM/Screener/internal/media/forwarding" + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/pion/interceptor" + "github.com/pion/webrtc/v4" +) + +func TestPublicationOwnsOneReservationAndSourceLifetime(t *testing.T) { + engine, err := NewEngine(EngineOptions{BindAddress: "127.0.0.1:0", IncludeLoopback: true}) + if err != nil { + t.Fatal(err) + } + defer engine.Close() + source, err := engine.NewSource("vp8", 3, 2, nil) + if err != nil { + t.Fatal(err) + } + defer source.Close() + for index := 0; index < 2; index++ { + if err = source.SetFormat(index, 320, 180); err != nil { + t.Fatal(err) + } + } + if err = source.ConfigureOutputs([]uint32{90_000, 300_000}); err != nil { + t.Fatal(err) + } + publication, err := engine.NewPublication(source, EdgeOptions{ConnectionID: "publication"}) + if err != nil { + t.Fatal(err) + } + candidate, err := engine.NewPublication(source, EdgeOptions{ConnectionID: "publication"}) + if err != nil { + t.Fatal(err) + } + if err = publication.transport.SetConnected(); err != nil { + t.Fatal(err) + } + if err = candidate.transport.SetConnected(); err != nil { + t.Fatal(err) + } + if len(source.media.GetDownTracks()) != 4 { + t.Fatalf("candidate overwrote current outputs: tracks=%d current=%s candidate=%s", + len(source.media.GetDownTracks()), publication.transport.PC.ID(), candidate.transport.PC.ID()) + } + if _, err = engine.NewPublication(source, EdgeOptions{ConnectionID: "third"}); err == nil { + t.Fatal("third publication was admitted") + } + if source.reservations != 2 { + t.Fatal("failed publication leaked its reservation") + } + edge, err := engine.NewEdge(source, EdgeOptions{ConnectionID: "sibling"}) + if err != nil { + t.Fatal(err) + } + defer edge.Close() + if _, err = engine.NewEdge(source, EdgeOptions{ConnectionID: "overflow"}); !errors.Is(err, ErrSourceCapacity) { + t.Fatal("publication did not consume exactly one outbound copy") + } + if err = candidate.Close(); err != nil { + t.Fatal(err) + } + if source.reservations != 1 || len(source.publications) != 1 || publication.signaling.connection.ConnectionState() == webrtc.PeerConnectionStateClosed { + t.Fatal("candidate rollback retired the current publication") + } + replacement, err := engine.NewPublication(source, EdgeOptions{ConnectionID: "replacement"}) + if err != nil { + t.Fatal(err) + } + if err = source.DisableLayer(0); err != nil { + t.Fatal(err) + } + if _, err = engine.NewPublication(source, EdgeOptions{ConnectionID: "failed-output"}); err == nil { + t.Fatal("failed output was admitted into publication metadata") + } + if source.reservations != 0 || len(source.publications) != 0 { + t.Fatal("failed publication demand retained a reservation") + } + if err = source.ConfigureOutputs([]uint32{90_000, 300_000}); err != nil { + t.Fatal(err) + } + replacement, err = engine.NewPublication(source, EdgeOptions{ConnectionID: "restored-output"}) + if err != nil { + t.Fatalf("validated replacement did not restore publication admission: %v", err) + } + if err = source.Close(); err != nil { + t.Fatal(err) + } + if replacement.signaling.connection.ConnectionState() != webrtc.PeerConnectionStateClosed || len(engine.publications) != 0 || source.reservations != 0 { + t.Fatal("source retirement did not release the publication") + } +} + +func TestPublicationQualityUsesHighestSentLayerCadence(t *testing.T) { + check := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + engine, err := NewEngine(EngineOptions{BindAddress: "127.0.0.1:0", IncludeLoopback: true}) + check(err) + defer engine.Close() + source, err := engine.NewSource("vp8", 1, 2, nil) + check(err) + defer source.Close() + check(source.SetFormat(0, 320, 180)) + check(source.SetFormat(1, 640, 360)) + check(source.ConfigureOutputs([]uint32{90_000, 300_000})) + publication, err := engine.NewPublication(source, EdgeOptions{ConnectionID: "publication_quality"}) + check(err) + mediaEngine := &webrtc.MediaEngine{} + check(mediaEngine.RegisterDefaultCodecs()) + for _, uri := range []string{"urn:ietf:params:rtp-hdrext:sdes:mid", "urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id"} { + check(mediaEngine.RegisterHeaderExtension(webrtc.RTPHeaderExtensionCapability{URI: uri}, webrtc.RTPCodecTypeVideo)) + } + registry := &interceptor.Registry{} + check(webrtc.RegisterDefaultInterceptors(mediaEngine, registry)) + settings := webrtc.SettingEngine{} + settings.SetIncludeLoopbackCandidate(true) + receiver, err := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine), webrtc.WithInterceptorRegistry(registry), webrtc.WithSettingEngine(settings)).NewPeerConnection(webrtc.Configuration{}) + check(err) + defer receiver.Close() + receiver.OnTrack(func(track *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { + for { + if _, _, readErr := track.ReadRTP(); readErr != nil { + return + } + } + }) + connectEdgeToReceiver(t, &publication.signaling, receiver) + assigned, err := forwarding.NewEncodedSource(forwarding.SourceOptions{ + ID: string(source.media.TrackID()), StreamID: source.media.StreamID(), Codec: source.media.Codec(), MaxPackets: encoded.MaxPacketWindow, + Formats: []forwarding.LayerFormat{{Width: 8, Height: 8, Bitrate: 45_000}, {Width: 8, Height: 8, Bitrate: 150_000}}, + }) + check(err) + defer assigned.Close() + check(assigned.BeginFrame(0, time.Now())) + for layer := range 2 { + check(assigned.WriteFrame(layer, encoded.Frame{Data: sfu.VP8KeyFrame8x8, Duration: time.Second / 30, Recovery: true})) + } + check(publication.transport.ReplaceSource(assigned.Source)) + metadata := publication.Media() + if len(metadata.Layers) != 2 || metadata.Layers[0].Width != 8 || metadata.Layers[0].Height != 8 || + metadata.Layers[0].Bitrate != 45_000 || metadata.Layers[1].Bitrate != 150_000 { + t.Fatalf("publication descriptor ignored assigned source: %+v", metadata) + } + now := time.Now() + if _, ok := publication.QualitySample(now); ok { + t.Fatal("first sample invented a window") + } + for frame := range 60 { + pts := time.Duration(frame+1) * time.Second / 30 + check(assigned.BeginFrame(pts, time.Now())) + check(assigned.WriteFrame(0, encoded.Frame{Data: sfu.VP8KeyFrame8x8, PTS: pts, Duration: time.Second / 30, Recovery: true})) + if frame%2 == 0 { + check(assigned.WriteFrame(1, encoded.Frame{Data: sfu.VP8KeyFrame8x8, PTS: pts, Duration: time.Second / 15, Recovery: true})) + } + time.Sleep(time.Second / 30) + } + sample, ok := publication.QualitySample(now.Add(2 * time.Second)) + if !ok || sample.RID != "h" || sample.ActiveVideoEncodingCount != 2 || sample.Width != 8 || sample.Height != 8 { + t.Fatalf("actual publication dimensions or active outputs missing: %+v counters=%+v", sample, publication.transport.Counters()) + } + if sample.IntervalFramesSent == 0 || sample.IntervalFramesSent != publication.qualityCounters.Frames[1] || + sample.FramesPerSecond != float64(publication.qualityCounters.Frames[1])/2 { + t.Fatalf("publication summed layer cadence instead of highest sent output: %+v", sample) + } + if sample.BitrateKbps != float64(publication.qualityCounters.VideoBytes*8)/2/1000 { + t.Fatal("publication did not aggregate actual written layer bytes") + } +} diff --git a/internal/client/mediaedge/quality.go b/internal/client/mediaedge/quality.go index 171660f..a69f420 100644 --- a/internal/client/mediaedge/quality.go +++ b/internal/client/mediaedge/quality.go @@ -1,39 +1,14 @@ package mediaedge import ( - "sync" "time" - "github.com/pion/interceptor" - "github.com/pion/interceptor/pkg/cc" - "github.com/pion/interceptor/pkg/gcc" + "github.com/TNTcraftHIM/Screener/internal/media/forwarding" "github.com/pion/webrtc/v4" ) const defaultNativeSourceInitialBitrate = 3_000_000 -type targetBitrateEstimator interface { - GetTargetBitrate() int - OnTargetBitrateChange(func(int)) -} - -type bandwidthObserver struct { - estimator targetBitrateEstimator - - mu sync.RWMutex - observed bool -} - -func newBandwidthObserver(estimator targetBitrateEstimator) *bandwidthObserver { - observer := &bandwidthObserver{estimator: estimator} - estimator.OnTargetBitrateChange(func(int) { - observer.mu.Lock() - observer.observed = true - observer.mu.Unlock() - }) - return observer -} - type QualitySample struct { SampleTimestampMs int64 SampleWindowMs int64 @@ -57,12 +32,12 @@ type qualityBaseline struct { } func (edge *Edge) QualitySample(now time.Time) (QualitySample, bool) { - video := edge.source.snapshot() + framesNow, bytesNow, format := edge.videoCounters() audioBytes := edge.audioSource.snapshotBytes() edge.qualityMu.Lock() previous := edge.qualityBaseline edge.qualityBaseline = qualityBaseline{ - at: now, frames: video.frames, videoBytes: video.bytes, audioBytes: audioBytes, + at: now, frames: framesNow, videoBytes: bytesNow, audioBytes: audioBytes, } edge.qualityMu.Unlock() if previous.at.IsZero() { @@ -70,27 +45,30 @@ func (edge *Edge) QualitySample(now time.Time) (QualitySample, bool) { } window := now.Sub(previous.at) if window < time.Second || window > 5*time.Second || - video.frames < previous.frames || video.bytes < previous.videoBytes || + framesNow < previous.frames || bytesNow < previous.videoBytes || audioBytes < previous.audioBytes { return QualitySample{}, false } - frames := video.frames - previous.frames - bytes := video.bytes - previous.videoBytes + audioBytes - previous.audioBytes + frames := framesNow - previous.frames + bytes := bytesNow - previous.videoBytes + audioBytes - previous.audioBytes seconds := window.Seconds() + if edge.audioSource != nil && edge.audioSource.encoder == nil { + edge.transport.SetAudioBitrate(uint32(float64((audioBytes-previous.audioBytes)*8) / seconds)) + } bitrate := float64(bytes*8) / seconds sample := QualitySample{ SampleTimestampMs: now.UnixMilli(), SampleWindowMs: window.Milliseconds(), RTPStatsID: edge.connection.ID(), - TrackIdentifier: edge.source.track.ID(), + TrackIdentifier: edge.transport.Output.ID(), State: "unknown", IntervalFramesEncoded: frames, FramesPerSecond: float64(frames) / seconds, BitrateKbps: bitrate / 1000, - Width: video.width, - Height: video.height, + Width: uint32(format >> 32), + Height: uint32(format), } - target, observed := edge.bandwidth.targetBitrate() + target, observed := edge.targetBitrate() if edge.State() != webrtc.PeerConnectionStateConnected || !observed || frames == 0 || bytes == 0 { return sample, true @@ -98,7 +76,7 @@ func (edge *Edge) QualitySample(now time.Time) (QualitySample, bool) { sample.AvailableOutgoingKbps = float64(target) / 1000 reason := "none" sample.State = "healthy" - if float64(target) < bitrate { + if edge.transport.Output.IsDeficient() || float64(target) < bitrate { reason = "bandwidth" sample.State = "degraded" } @@ -106,55 +84,30 @@ func (edge *Edge) QualitySample(now time.Time) (QualitySample, bool) { return sample, true } -func (observer *bandwidthObserver) targetBitrate() (int, bool) { - observer.mu.RLock() - observed := observer.observed - observer.mu.RUnlock() - if !observed { - return 0, false +func (edge *Edge) videoCounters() (frames, bytes, format uint64) { + stats := edge.transport.Output.GetState().RTPStats.ToProto() + if stats != nil { + frames, bytes = uint64(stats.Frames), stats.Bytes-stats.HeaderBytes } - return observer.estimator.GetTargetBitrate(), true + current := edge.transport.Output.State().Current + width, height := sentVideoDimensions(edge.transport.CurrentSource(), int(current)) + format = uint64(width)<<32 | uint64(height) + return } -type bandwidthObservers struct { - mu sync.Mutex - pending map[string]*bandwidthObserver -} - -func configureBandwidthObservers( - media *webrtc.MediaEngine, - registry *interceptor.Registry, - initialBitrate int, -) (*bandwidthObservers, error) { - if initialBitrate <= 0 { - initialBitrate = defaultNativeSourceInitialBitrate +func sentVideoDimensions(source *forwarding.Source, layer int) (uint32, uint32) { + if layer < 0 { + return 0, 0 } - observers := &bandwidthObservers{pending: make(map[string]*bandwidthObserver)} - factory, err := cc.NewInterceptor(func() (cc.BandwidthEstimator, error) { - return gcc.NewSendSideBWE( - gcc.SendSideBWEInitialBitrate(initialBitrate), - gcc.SendSideBWEPacer(gcc.NewNoOpPacer()), - ) - }) - if err != nil { - return nil, err + sizes := source.VideoSizes() + if layer < len(sizes) { + return sizes[layer].Width, sizes[layer].Height } - factory.OnNewPeerConnection(func(id string, estimator cc.BandwidthEstimator) { - observers.mu.Lock() - observers.pending[id] = newBandwidthObserver(estimator) - observers.mu.Unlock() - }) - registry.Add(factory) - if err = webrtc.ConfigureTWCCHeaderExtensionSender(media, registry); err != nil { - return nil, err + // VideoSizes stops at an unproduced lower slot. The source owner also + // publishes actual dimensions in metadata as encoded frames arrive. + info := source.TrackInfo() + if layer < len(info.Layers) { + return info.Layers[layer].Width, info.Layers[layer].Height } - return observers, nil -} - -func (observers *bandwidthObservers) take(id string) *bandwidthObserver { - observers.mu.Lock() - defer observers.mu.Unlock() - observer := observers.pending[id] - delete(observers.pending, id) - return observer + return 0, 0 } diff --git a/internal/client/mediaedge/quality_test.go b/internal/client/mediaedge/quality_test.go index cd7cf08..608e7fb 100644 --- a/internal/client/mediaedge/quality_test.go +++ b/internal/client/mediaedge/quality_test.go @@ -3,18 +3,58 @@ package mediaedge import ( "testing" "time" + + "github.com/TNTcraftHIM/Screener/internal/media/encoded" + "github.com/TNTcraftHIM/Screener/internal/media/forwarding" + "github.com/livekit/livekit-server/pkg/sfu" ) -type fixedTargetEstimator struct { - target int +type qualityAllocationReceiver struct{ sfu.TrackReceiver } + +func (receiver qualityAllocationReceiver) GetLayeredBitrate() ([]int32, sfu.Bitrates) { + return []int32{0, 1}, sfu.Bitrates{{100_000}, {900_000}} } -func (estimator fixedTargetEstimator) GetTargetBitrate() int { - return estimator.target -} - -func (estimator fixedTargetEstimator) OnTargetBitrateChange(callback func(int)) { - callback(estimator.target) +func TestQualityDimensionsFollowAssignedSource(t *testing.T) { + check := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + engine, err := NewEngine(EngineOptions{BindAddress: "127.0.0.1:0", IncludeLoopback: true}) + check(err) + defer engine.Close() + source, err := engine.NewSource("vp8", 1, 2, nil) + check(err) + defer source.Close() + check(source.SetFormat(0, 320, 180)) + check(source.SetFormat(1, 640, 360)) + check(source.ConfigureOutputs([]uint32{90_000, 300_000})) + edge, receiver, _ := connectedReceiver(t, engine, source, "assigned-quality") + defer receiver.Close() + assigned, err := forwarding.NewEncodedSource(forwarding.SourceOptions{ + ID: string(source.media.TrackID()), StreamID: source.media.StreamID(), Codec: source.media.Codec(), MaxPackets: encoded.MaxPacketWindow, + Formats: []forwarding.LayerFormat{{Width: 4, Height: 4, Bitrate: 45_000}, {Width: 8, Height: 8, Bitrate: 150_000}}, + }) + check(err) + defer assigned.Close() + check(assigned.BeginFrame(0, time.Now())) + check(assigned.WriteFrame(1, encoded.Frame{Data: sfu.VP8KeyFrame8x8, Duration: time.Second / 30, Recovery: true})) + check(edge.transport.ReplaceSource(assigned.Source)) + deadline := time.Now().Add(3 * time.Second) + for frame := 1; time.Now().Before(deadline); frame++ { + pts := time.Duration(frame) * time.Second / 30 + check(assigned.BeginFrame(pts, time.Now())) + check(assigned.WriteFrame(1, encoded.Frame{Data: sfu.VP8KeyFrame8x8, PTS: pts, Duration: time.Second / 30, Recovery: true})) + frames, _, format := edge.videoCounters() + if frames > 0 && format == uint64(8)<<32|8 { + return + } + time.Sleep(time.Second / 30) + } + _, _, format := edge.videoCounters() + t.Fatalf("quality dimensions still refer to original source: %dx%d", uint32(format>>32), uint32(format)) } func TestQualitySampleUsesEstimatorCapacityAgainstEncodedPayload(t *testing.T) { @@ -25,20 +65,30 @@ func TestQualitySampleUsesEstimatorCapacityAgainstEncodedPayload(t *testing.T) { t.Fatal(err) } t.Cleanup(func() { _ = engine.Close() }) - source, err := engine.NewSource("h264", 1, nil) + source, err := engine.NewSource("h264", 1, 2, nil) if err != nil { t.Fatal(err) } edge, receiver, _ := connectedReceiver(t, engine, source, "limited-edge") t.Cleanup(func() { _ = receiver.Close() }) - edge.bandwidth = newBandwidthObserver(fixedTargetEstimator{target: 100_000}) + edge.targetBitrate = func() (int, bool) { return 100_000, true } // Encoded relay sources need no decoded dimensions to compare payload and capacity. started := time.Now() if _, ok := edge.QualitySample(started); ok { t.Fatal("first quality sample did not establish a baseline") } - source.frames.Add(60) - source.bytes.Add(1_000_000) + stats := edge.transport.Output.GetState().RTPStats + stats.SetClockRate(videoClockRate) + for frame := 0; frame < 60; frame++ { + size := 16_666 + if frame == 59 { + size += 40 + } + stats.Update(started.Add(time.Duration(frame)*time.Second/30).UnixNano(), + uint64(frame+1), uint64(frame*3000), true, 12, size, 0, false) + } + // The library does not publish a zero-duration statistics snapshot. + time.Sleep(time.Millisecond) sample, ok := edge.QualitySample(started.Add(2 * time.Second)) if !ok || sample.State != "degraded" || sample.Reason == nil || @@ -47,4 +97,40 @@ func TestQualitySampleUsesEstimatorCapacityAgainstEncodedPayload(t *testing.T) { sample.AvailableOutgoingKbps != 100 { t.Fatalf("quality sample = %+v, %v", sample, ok) } + + // Real library allocation remains deficient after selecting a layer that + // fits the link; steady low delivery must not cancel quality convergence. + output := edge.transport.Output + output.SetReceiver(qualityAllocationReceiver{TrackReceiver: output.Receiver()}) + output.UpTrackMaxPublishedLayerChange(1) + output.UpTrackMaxTemporalLayerSeenChange(0) + output.SetMaxSpatialLayer(1) + for index, check := range []struct { + budget, sent int + layer int32 + state string + }{ + {200_000, 100_000, 0, "degraded"}, + {1_000_000, 900_000, 1, "healthy"}, + } { + output.SetBudget(int64(check.budget)) + edge.targetBitrate = func() (int, bool) { return check.budget, true } + if output.State().Target != check.layer || output.IsDeficient() != (check.state == "degraded") { + t.Fatalf("fixture did not establish the expected committed allocation: %+v", output.State()) + } + bytes := check.sent / 4 + for frame := 0; frame < 60; frame++ { + number := uint64((index+1)*60 + frame + 1) + size := bytes / 60 + if frame == 59 { + size += bytes % 60 + } + stats.Update(started.Add(time.Duration(number)*time.Second/30).UnixNano(), number, + number*3000, true, 12, size, 0, false) + } + sample, ok = edge.QualitySample(started.Add(time.Duration(index+2) * 2 * time.Second)) + if !ok || sample.State != check.state || sample.BitrateKbps != float64(check.sent)/1000 { + t.Fatalf("allocation %d quality = %+v, %v", index, sample, ok) + } + } } diff --git a/internal/client/mediaedge/receiver.go b/internal/client/mediaedge/receiver.go index 9d94534..85e7579 100644 --- a/internal/client/mediaedge/receiver.go +++ b/internal/client/mediaedge/receiver.go @@ -3,6 +3,7 @@ package mediaedge import ( "errors" "io" + "log/slog" "strings" "sync" @@ -21,6 +22,7 @@ type ReceiverOptions struct { ICEServers []webrtc.ICEServer EdgeCapacity int Events ReceiverEvents + Relay *RelayOptions } // Receiver owns one inbound WebRTC connection and exposes its encoded media as @@ -54,7 +56,6 @@ func (engine *Engine) NewReceiver(options ReceiverOptions) (*Receiver, webrtc.Se if err != nil { return nil, webrtc.SessionDescription{}, err } - _ = engine.bandwidth.take(connection.ID()) if err = connection.SetRemoteDescription(options.Offer); err != nil { _ = connection.Close() return nil, webrtc.SessionDescription{}, err @@ -68,11 +69,18 @@ func (engine *Engine) NewReceiver(options ReceiverOptions) (*Receiver, webrtc.Se connection: connection, events: options.Events, } - receiver.source, err = engine.NewSource(videoCodec, options.EdgeCapacity, receiver.RequestKeyFrame) + layers := 1 + if _, supported := relayBackend(videoCodec, options.Relay); supported { + layers = 2 + } + receiver.source, err = engine.NewSource(videoCodec, options.EdgeCapacity, layers, receiver.RequestKeyFrame) if err != nil { _ = connection.Close() return nil, webrtc.SessionDescription{}, err } + if layers > 1 { + receiver.source.relay = &relayDerivation{source: receiver.source, options: *options.Relay} + } if hasAudio { receiver.audioSource, err = engine.NewRelayedAudioSource(options.EdgeCapacity) if err != nil { @@ -89,19 +97,26 @@ func (engine *Engine) NewReceiver(options ReceiverOptions) (*Receiver, webrtc.Se ) connection.OnICECandidate(receiver.localCandidates.addPion) connection.OnConnectionStateChange(func(state webrtc.PeerConnectionState) { - if receiver.events.ConnectionState == nil { - return + debug := slog.Default().Enabled(engine.ctx, slog.LevelDebug) + if debug { + slog.Debug("screener-client", "event", "media-connection-state", "direction", "inbound", "state", state.String()) } var selected *SelectedPair - if state == webrtc.PeerConnectionStateConnected { + if state == webrtc.PeerConnectionStateConnected && (receiver.events.ConnectionState != nil || debug) { if pair, pairErr := receiver.SelectedPair(); pairErr == nil { selected = &pair + if debug { + slog.Debug("screener-client", "event", "media-selected-path", "direction", "inbound", + "localType", pair.Local.String(), "remoteType", pair.Remote.String(), "natTraversalPath", pair.NatTraversalPath) + } } } - receiver.events.ConnectionState(state, selected) + if receiver.events.ConnectionState != nil { + receiver.events.ConnectionState(state, selected) + } }) connection.OnTrack(func(track *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver) { - go receiver.readRTCP(rtpReceiver) + go receiver.readRTCP(track, rtpReceiver) receiver.consumeTrack(track, rtpReceiver) }) answer, err := receiver.createAnswer() @@ -243,14 +258,22 @@ func (receiver *Receiver) consumeTrack(track *webrtc.TrackRemote, _ *webrtc.RTPR } } -func (receiver *Receiver) readRTCP(rtpReceiver *webrtc.RTPReceiver) { +func (receiver *Receiver) readRTCP(track *webrtc.TrackRemote, rtpReceiver *webrtc.RTPReceiver) { if rtpReceiver == nil { return } for { - if _, _, err := rtpReceiver.ReadRTCP(); err != nil { + packets, _, err := rtpReceiver.ReadRTCP() + if err != nil { return } + if track.Kind() == webrtc.RTPCodecTypeVideo { + for _, packet := range packets { + if report, ok := packet.(*rtcp.SenderReport); ok && report.SSRC == uint32(track.SSRC()) { + _ = receiver.source.SenderReport(report) + } + } + } } } diff --git a/internal/client/mediaedge/receiver_test.go b/internal/client/mediaedge/receiver_test.go index c5c4c18..6c96135 100644 --- a/internal/client/mediaedge/receiver_test.go +++ b/internal/client/mediaedge/receiver_test.go @@ -2,9 +2,11 @@ package mediaedge import ( "strings" + "sync" "testing" "time" + "github.com/livekit/livekit-server/pkg/sfu" "github.com/pion/rtp" "github.com/pion/webrtc/v4" ) @@ -39,19 +41,10 @@ func TestReceiverCodecMatchesTheSingleNegotiatedAnswer(t *testing.T) { if err != nil { t.Fatal(err) } - engine.bandwidth.mu.Lock() - pendingBefore := len(engine.bandwidth.pending) - engine.bandwidth.mu.Unlock() receiver, answer, err := engine.NewReceiver(ReceiverOptions{Offer: offer, EdgeCapacity: 1}) if err != nil { t.Fatal(err) } - engine.bandwidth.mu.Lock() - pendingObservers := len(engine.bandwidth.pending) - engine.bandwidth.mu.Unlock() - if pendingObservers != pendingBefore { - t.Fatalf("receiver changed pending bandwidth observers from %d to %d", pendingBefore, pendingObservers) - } t.Cleanup(func() { _ = receiver.Close() }) if receiver.Codec() != want || !strings.Contains(strings.ToLower(answer.SDP), want+"/90000") || (strings.Contains(strings.ToLower(answer.SDP), "vp8/90000") && strings.Contains(strings.ToLower(answer.SDP), "h264/90000")) { @@ -117,17 +110,22 @@ func testReceiverForwarding(t *testing.T, codec string) { var nativeReceiver *Receiver var upstreamCandidates []*webrtc.ICECandidateInit var receiverCandidates []*webrtc.ICECandidateInit + var candidateMu sync.Mutex remoteReady := false upstream.OnICECandidate(func(candidate *webrtc.ICECandidate) { if candidate == nil { return } value := candidate.ToJSON() - if nativeReceiver != nil && remoteReady { - _ = nativeReceiver.AddRemoteCandidate(&value) - } else { + candidateMu.Lock() + ready := remoteReady + if !ready { upstreamCandidates = append(upstreamCandidates, &value) } + candidateMu.Unlock() + if ready { + _ = nativeReceiver.AddRemoteCandidate(&value) + } }) upstreamGathered := webrtc.GatheringCompletePromise(upstream) offer, err := upstream.CreateOffer(nil) @@ -144,12 +142,17 @@ func testReceiverForwarding(t *testing.T, codec string) { EdgeCapacity: 1, Events: ReceiverEvents{ LocalCandidate: func(candidate *webrtc.ICECandidateInit) { - if candidate != nil { - if remoteReady { - _ = upstream.AddICECandidate(*candidate) - } else { - receiverCandidates = append(receiverCandidates, candidate) - } + if candidate == nil { + return + } + candidateMu.Lock() + ready := remoteReady + if !ready { + receiverCandidates = append(receiverCandidates, candidate) + } + candidateMu.Unlock() + if ready { + _ = upstream.AddICECandidate(*candidate) } }, }, @@ -164,11 +167,15 @@ func testReceiverForwarding(t *testing.T, codec string) { if err = upstream.SetRemoteDescription(answer); err != nil { t.Fatal(err) } + candidateMu.Lock() remoteReady = true - for _, candidate := range upstreamCandidates { + pendingUpstream, pendingReceiver := upstreamCandidates, receiverCandidates + upstreamCandidates, receiverCandidates = nil, nil + candidateMu.Unlock() + for _, candidate := range pendingUpstream { _ = nativeReceiver.AddRemoteCandidate(candidate) } - for _, candidate := range receiverCandidates { + for _, candidate := range pendingReceiver { _ = upstream.AddICECandidate(*candidate) } waitConnected(t, upstream, "native receiver") @@ -193,6 +200,12 @@ func testReceiverForwarding(t *testing.T, codec string) { } if codec == "vp8" { want.Payload = []byte{0x10, 0x10, 0, 0, 0x9d, 0x01, 0x2a, 0x80, 0x02, 0xe0, 0x01, 0} + } else { + want.Payload = []byte{0x78} + for _, nalu := range sfu.H264KeyFrame2x2 { + want.Payload = append(want.Payload, byte(len(nalu)>>8), byte(len(nalu))) + want.Payload = append(want.Payload, nalu...) + } } if err = want.SetExtension(9, []byte{0xde, 0xad}); err != nil { t.Fatal(err) @@ -235,8 +248,10 @@ func testReceiverForwarding(t *testing.T, codec string) { if err = input.track.WriteRTP(padding); err != nil { t.Fatal(err) } - if packet := waitPacket(t, input.packets); !packet.Padding || packet.PaddingSize != padding.PaddingSize || len(packet.Payload) != 0 { - t.Fatalf("stream %d lost RTP padding continuity", index) + if index == 1 { + if packet := waitPacket(t, input.packets); !packet.Padding || packet.PaddingSize != padding.PaddingSize || len(packet.Payload) != 0 { + t.Fatal("audio lost RTP padding continuity") + } } next := input.media.Clone() next.SequenceNumber += 2 @@ -244,7 +259,11 @@ func testReceiverForwarding(t *testing.T, codec string) { if err = input.track.WriteRTP(next); err != nil { t.Fatal(err) } - if packet := waitPacket(t, input.packets); string(packet.Payload) != string(next.Payload) { + packet := waitPacket(t, input.packets) + if index == 0 && packet.SequenceNumber != got.SequenceNumber+1 { + t.Fatal("video projection did not close the filtered padding sequence gap") + } + if string(packet.Payload) != string(next.Payload) { t.Fatalf("stream %d stopped delivering media after padding", index) } } diff --git a/internal/client/mediaedge/relay_chain_fixture_test.go b/internal/client/mediaedge/relay_chain_fixture_test.go new file mode 100644 index 0000000..142a8f5 --- /dev/null +++ b/internal/client/mediaedge/relay_chain_fixture_test.go @@ -0,0 +1,299 @@ +package mediaedge + +import ( + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "os" + "sync" + "testing" + "time" + + "github.com/TNTcraftHIM/Screener/internal/client/nativecapture" + "github.com/TNTcraftHIM/Screener/internal/media/encoded" + "github.com/pion/webrtc/v4" +) + +// Replays actual SDK encoder output through independent Pion connections. The +// relays retain derivation capability, but healthy children need only raw reuse. +func TestRelayChainEncodedFixture(t *testing.T) { + fixture := os.Getenv("SCREENER_POOL_CHAIN_FIXTURE") + if fixture == "" { + t.Skip("set SCREENER_POOL_CHAIN_FIXTURE and SCREENER_NATIVE_CAPTURE") + } + check := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + executable := os.Getenv("SCREENER_NATIVE_CAPTURE") + info, err := os.Stat(executable) + check(err) + if info.IsDir() { + t.Fatal("SCREENER_NATIVE_CAPTURE must name the existing capture executable") + } + input, err := os.Open(fixture) + check(err) + defer input.Close() + info, err = input.Stat() + check(err) + if info.Size() > 64<<20 { + t.Fatal("encoded chain fixture exceeds 64 MiB") + } + type fixtureFrame struct { + Index, Width, Height int + Recovery bool + DataHex string + Data []byte `json:"-"` + } + var frames []fixtureFrame + identities := make(map[[sha256.Size]byte][]int) + decoder := json.NewDecoder(input) + for { + var frame fixtureFrame + if err = decoder.Decode(&frame); err == io.EOF { + break + } + check(err) + if len(frames) >= 400 || frame.Index != len(frames) || frame.Width <= 0 || frame.Height <= 0 || + frame.Width > 2560 || frame.Height > 1440 || len(frame.DataHex) == 0 || len(frame.DataHex) > 8<<20 || + len(frame.DataHex)%2 != 0 { + t.Fatal("encoded chain fixture has invalid or excessive frames") + } + frame.Data, err = hex.DecodeString(frame.DataHex) + check(err) + frame.DataHex = "" + if len(frame.Data) < 3 || (frame.Data[0]&1 == 0) != frame.Recovery { + t.Fatal("fixture recovery metadata does not match VP8") + } + if len(frames) > 0 && (frame.Width != frames[0].Width || frame.Height != frames[0].Height) { + t.Fatal("encoded chain fixture must retain one format") + } + hash := sha256.Sum256(frame.Data) + identities[hash] = append(identities[hash], frame.Index) + frames = append(frames, frame) + } + if len(frames) < 180 || !frames[0].Recovery { + t.Fatal("encoded chain fixture needs at least 180 frames beginning with recovery") + } + + var engines []*Engine + var edges []*Edge + var relays []*Receiver + var leaves []*webrtc.PeerConnection + closeAll := func() { + for _, leaf := range leaves { + if closeErr := leaf.Close(); closeErr != nil { + t.Error(closeErr) + } + } + for i := len(relays) - 1; i >= 0; i-- { + if closeErr := relays[i].Close(); closeErr != nil { + t.Error(closeErr) + } + } + for _, edge := range edges { + if closeErr := edge.Close(); closeErr != nil { + t.Error(closeErr) + } + } + for _, engine := range engines { + if closeErr := engine.Close(); closeErr != nil { + t.Error(closeErr) + } + } + } + t.Cleanup(closeAll) + for range 3 { + engine, createErr := NewEngine(EngineOptions{ + BindAddress: "127.0.0.1:0", IncludeLoopback: true, InitialBitrate: 10_000_000, + }) + check(createErr) + engines = append(engines, engine) + } + host, err := engines[0].NewSource("vp8", 2, 1, nil) + check(err) + t.Cleanup(func() { _ = host.Close() }) + check(host.SetFormat(0, uint32(frames[0].Width), uint32(frames[0].Height))) + check(host.ConfigureOutputs([]uint32{2_000_000})) + sources := []*Source{host} + for hop := 0; hop < 2; hop++ { + edge, createErr := engines[hop].NewEdge(sources[hop], EdgeOptions{ConnectionID: fmt.Sprintf("chain-hop-%d", hop+1)}) + check(createErr) + edges = append(edges, edge) + gathered := webrtc.GatheringCompletePromise(edge.connection) + _, err = edge.CreateOffer() + check(err) + waitSignal(t, gathered, "chain offer gathering") + receiver, _, createErr := engines[hop+1].NewReceiver(ReceiverOptions{ + Offer: *edge.connection.LocalDescription(), EdgeCapacity: 2, + Relay: &RelayOptions{CaptureProcess: executable, Capabilities: nativecapture.Capabilities{ + SoftwareVP8: true, Adapters: []nativecapture.Adapter{{Index: 0}}, + }}, + }) + check(createErr) + relays = append(relays, receiver) + waitSignal(t, webrtc.GatheringCompletePromise(receiver.connection), "chain answer gathering") + check(edge.SetAnswer(*receiver.connection.LocalDescription())) + waitConnected(t, edge.connection, "chain upstream") + waitConnected(t, receiver.connection, "chain native receiver") + check(receiver.Source().SetRelayProfile(nativecapture.VideoProfile{ + Width: 1280, Height: 720, Framerate: 30, Bitrate: 2_000_000, Preference: "balanced", + })) + if receiver.Source().relay == nil { + t.Fatal("native relay derivation capability was not enabled") + } + sources = append(sources, receiver.Source()) + } + + type leafResult struct { + Frames int `json:"frames"` + FirstIndex int `json:"firstIndex"` + LastIndex int `json:"lastIndex"` + LastTimestamp uint32 `json:"-"` + SSRC uint32 `json:"-"` + } + var resultMu sync.Mutex + var results [3]leafResult + var leafDone [3]chan struct{} + errors := make(chan error, 1) + fail := func(err error) { + select { + case errors <- err: + default: + } + } + for leaf := range 3 { + edge, createErr := engines[leaf].NewEdge(sources[leaf], EdgeOptions{ConnectionID: fmt.Sprintf("chain-leaf-%d", leaf)}) + check(createErr) + edges = append(edges, edge) + receiver := newReceiver(t) + leaves = append(leaves, receiver) + leafDone[leaf] = make(chan struct{}) + receiver.OnTrack(func(track *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { + defer close(leafDone[leaf]) + video := relayVideoInput{codec: "vp8", onPacketDropped: func() { + fail(fmt.Errorf("leaf %d discarded an incomplete access unit", leaf)) + }} + for { + packet, _, readErr := track.ReadRTP() + if readErr != nil { + return + } + if err := video.push(packet); err != nil { + fail(err) + return + } + for { + sample, timestamp := video.pop() + if sample == nil { + break + } + // SampleBuilder removes the hop-specific VP8 RTP descriptor. + candidates := identities[sha256.Sum256(sample.Data)] + resultMu.Lock() + result := &results[leaf] + matched := -1 + for _, index := range candidates { + if result.Frames == 0 || index > result.LastIndex { + matched = index + break + } + } + if matched < 0 || result.Frames == 0 && !frames[matched].Recovery || + result.Frames > 0 && (matched != result.LastIndex+1 || + int32(timestamp-result.LastTimestamp) <= 0 || result.SSRC != packet.SSRC) { + resultMu.Unlock() + fail(fmt.Errorf("leaf %d received altered, repeated, or non-monotonic encoded media", leaf)) + return + } + if result.Frames == 0 { + result.FirstIndex = matched + } + result.Frames++ + result.LastIndex, result.LastTimestamp, result.SSRC = matched, timestamp, packet.SSRC + resultMu.Unlock() + } + } + }) + connectEdgeToReceiver(t, edge, receiver) + } + checkIdleRelays := func() { + for hop, receiver := range relays { + relay := receiver.Source().relay + relay.mu.Lock() + started := relay.run != nil + relay.mu.Unlock() + if started { + t.Fatalf("relay %d started derivation for a healthy reused representation", hop+1) + } + } + } + period := time.Second / 30 + ticker := time.NewTicker(period) + defer ticker.Stop() + for index, frame := range frames { + select { + case <-ticker.C: + case readErr := <-errors: + t.Fatal(readErr) + } + pts := time.Duration(index) * period + _, err = host.BeginFrame(pts) + check(err) + check(host.WriteVideo(0, encoded.Frame{Data: frame.Data, PTS: pts, Duration: period, Recovery: frame.Recovery})) + checkIdleRelays() + } + select { + case readErr := <-errors: + t.Fatal(readErr) + case <-time.After(500 * time.Millisecond): + } + checkIdleRelays() + closeAll() + check(host.Close()) + for _, done := range leafDone { + waitSignal(t, done, "chain leaf reader shutdown") + } + select { + case readErr := <-errors: + t.Fatal(readErr) + default: + } + resultMu.Lock() + actual := results + resultMu.Unlock() + for _, receiver := range relays { + check(receiver.connection.GracefulClose()) + if receiver.connection.ConnectionState() != webrtc.PeerConnectionStateClosed { + t.Fatal("chain retained a native inbound PeerConnection after cleanup") + } + } + for leaf, result := range actual { + if result.Frames < 120 { + t.Fatalf("leaf %d has insufficient encoded reuse: %+v", leaf, result) + } + } + for _, edge := range edges { + // Close can return while a concurrent close is still retiring transport. + check(edge.connection.GracefulClose()) + if edge.connection.ConnectionState() != webrtc.PeerConnectionStateClosed { + t.Fatal("chain retained an outbound PeerConnection after cleanup") + } + } + summary, err := json.Marshal(struct { + InputFrames int `json:"inputFrames"` + RelayHops int `json:"relayHops"` + Leaves [3]leafResult `json:"leaves"` + DerivationObserved bool `json:"derivationObserved"` + Cleanup bool `json:"cleanup"` + }{len(frames), 2, actual, false, true}) + check(err) + t.Log(string(summary)) + if output := os.Getenv("SCREENER_POOL_CHAIN_OUTPUT"); output != "" { + check(os.WriteFile(output, append(summary, '\n'), 0o600)) + } +} diff --git a/internal/client/mediaedge/relay_derivation.go b/internal/client/mediaedge/relay_derivation.go new file mode 100644 index 0000000..4ad4625 --- /dev/null +++ b/internal/client/mediaedge/relay_derivation.go @@ -0,0 +1,422 @@ +package mediaedge + +import ( + "context" + "errors" + "io" + "sync" + "sync/atomic" + "time" + + "github.com/TNTcraftHIM/Screener/internal/client/nativecapture" + "github.com/TNTcraftHIM/Screener/internal/media/encoded" + "github.com/TNTcraftHIM/Screener/internal/media/forwarding" + "github.com/livekit/mediatransportutil/pkg/utils" + "github.com/pion/rtp" +) + +const relayPacketLimit = encoded.MaxPacketWindow + +type RelayOptions struct { + CaptureProcess string + Capabilities nativecapture.Capabilities +} + +func relayBackend(codec string, options *RelayOptions) (nativecapture.EncodedVideoOptions, bool) { + backend := nativecapture.EncodedVideoOptions{Codec: codec} + if options == nil || options.CaptureProcess == "" { + return backend, false + } + for _, adapter := range options.Capabilities.Adapters { + if codec == "vp8" && options.Capabilities.SoftwareVP8 { + backend.AdapterIndex = adapter.Index + return backend, true + } + if codec == "h264" && len(adapter.HardwareH264) > 0 { + backend.AdapterIndex, backend.EncoderIndex = adapter.Index, adapter.HardwareH264[0].Index + return backend, true + } + } + return backend, false +} + +// relayDerivation owns only the optional missing lower representation. Raw RTP +// never waits for it. A retired process must exit before another one can start. +type relayDerivation struct { + source *Source + options RelayOptions + recovery atomic.Bool + mu sync.Mutex + run *relayRun +} + +type relayPlan struct { + profile *nativecapture.VideoProfile + output nativecapture.OutputProfile + format uint64 + controls OutputPlan + wanted bool +} + +type relayRun struct { + owner *relayDerivation + plan relayPlan + ctx context.Context + cancel context.CancelCauseFunc + done chan struct{} + packets chan *rtp.Packet + anchorRTP uint32 + anchorTime time.Time +} + +// Called with writeMu held. Declared ceilings are not measured input quality. +func (source *Source) configureRelayFormats(reset bool) error { + source.mu.Lock() + defer source.mu.Unlock() + if source.closed { + return io.ErrClosedPipe + } + if source.relayProfile == nil { + return nil + } + base := len(source.formats) - 1 + format := source.formats[base].Load() + width, height := uint32(format>>32), uint32(format) + outputs := nativecapture.ScreenShareOutputs(*source.relayProfile) + lower := outputs[0] + if width > 0 && height > 0 { + lower.Width, lower.Height = min(width/4*2, lower.Width), min(height/4*2, lower.Height) + } + if len(source.outputBitrates) != 2+source.capacity { + source.outputBitrates = make([]uint32, 2+source.capacity) + reset = true + } + for slot := range source.outputBitrates { + if reset || source.outputBitrates[slot] != 0 { + source.outputBitrates[slot] = lower.Bitrate + } + } + source.outputBitrates[1] = source.relayProfile.Bitrate + formats := []forwarding.LayerFormat{ + {Width: lower.Width, Height: lower.Height, Bitrate: lower.Bitrate}, + {Width: width, Height: height, Bitrate: source.relayProfile.Bitrate}, + } + if err := source.media.UpdateFormats(formats); err != nil { + return err + } + for slot, group := range source.groups { + if slot != 0 { + if err := group.media.UpdateFormats(formats); err != nil { + return err + } + } + } + return nil +} + +func (source *Source) relayPlan() relayPlan { + source.writeMu.Lock() + defer source.writeMu.Unlock() + source.mu.Lock() + plan := relayPlan{} + if source.closed || source.relayProfile == nil || len(source.outputBitrates) == 0 { + source.mu.Unlock() + return plan + } + base := len(source.formats) - 1 + plan.profile, plan.format = source.relayProfile, source.formats[base].Load() + edges := make([]*Edge, 0, len(source.edges)) + for edge := range source.edges { + edges = append(edges, edge) + } + source.mu.Unlock() + width, height := uint32(plan.format>>32), uint32(plan.format) + plan.output = nativecapture.ScreenShareOutputs(*plan.profile)[0] + plan.output.Width = min(width/4*2, plan.output.Width) + plan.output.Height = min(height/4*2, plan.output.Height) + if !plan.output.Valid() || width > 2560 || height > 1440 { + return plan + } + demands := source.collectDemands(edges, nil) + for _, demand := range demands { + plan.wanted = plan.wanted || demand.active && demand.lower + } + plan.controls, _ = source.planGroups(demands) + return plan +} + +func (relay *relayDerivation) push(packet *rtp.Packet) { + relay.mu.Lock() + requestOriginal := false + defer func() { + relay.mu.Unlock() + if requestOriginal { + relay.source.RequestRecoveryFrame() + } + }() + if relay.run != nil { + select { + case <-relay.run.done: + relay.run = nil + default: + } + } + if packet.Marker { + plan := relay.source.relayPlan() + if !plan.wanted && relay.source.groupRecovery.Load()&2 != 0 { + relay.source.groupRecovery.And(^uint32(2)) + requestOriginal = true + } + if relay.run != nil && (!plan.wanted || plan.profile != relay.run.plan.profile || plan.format != relay.run.plan.format) { + relay.run.cancel(nil) + return + } + if relay.run == nil && plan.wanted { + at, available := relay.source.media.ReferenceTime(len(relay.source.formats)-1, packet.Timestamp) + if !available { + return + } + ctx, cancel := context.WithCancelCause(relay.source.engine.ctx) + relay.run = &relayRun{ + owner: relay, plan: plan, ctx: ctx, cancel: cancel, + done: make(chan struct{}), packets: make(chan *rtp.Packet, relayPacketLimit), + anchorRTP: packet.Timestamp, anchorTime: at, + } + relay.recovery.Store(true) + go relay.run.serve() + // Earlier packets of this frame used the raw path only. Begin assembly + // with the next complete frame, not this frame's trailing fragment. + return + } + } + if relay.run == nil { + return + } + select { + case relay.run.packets <- packet.Clone(): + case <-relay.run.ctx.Done(): + default: + // Discarding a compressed reference frame silently poisons the decoder. + // Retire this derived output; the existing edge owner handles its failure. + relay.run.cancel(errors.New("native relay decoder input is full")) + } +} + +func (relay *relayDerivation) cancel() { + relay.mu.Lock() + if relay.run != nil { + relay.run.cancel(nil) + } + relay.mu.Unlock() +} + +func (relay *relayDerivation) close() { + relay.mu.Lock() + run := relay.run + if run != nil { + run.cancel(nil) + } + relay.mu.Unlock() + if run != nil { + <-run.done + } +} + +func (run *relayRun) serve() { + defer close(run.done) + defer func() { + if cause := context.Cause(run.ctx); cause != nil && !errors.Is(cause, context.Canceled) { + for slot := 0; slot < 2+run.owner.source.capacity; slot++ { + if slot == 1 { + continue + } + retire, _ := run.owner.source.markLayerUnavailable(slot, run) + if retire != nil { + retire() + } + } + } + }() + backend, _ := relayBackend(run.owner.source.codec, &run.owner.options) + backend.Preference = run.plan.profile.Preference + backend.Outputs = make([]nativecapture.OutputProfile, run.owner.source.capacity+1) + for slot := range backend.Outputs { + backend.Outputs[slot] = run.plan.output + } + stream, err := nativecapture.StartEncodedVideo(run.ctx, run.owner.options.CaptureProcess, backend) + if err != nil { + run.cancel(err) + return + } + run.owner.source.SetLowestLayerRateControlled(true) + defer run.owner.source.SetLowestLayerRateControlled(false) + run.owner.source.BeginGeneration() + readDone := make(chan struct{}) + go func() { + defer close(readDone) + run.read(stream) + }() + run.owner.source.media.SendPLI(int32(len(run.owner.source.formats)-1), false) + if err = run.write(stream); err != nil { + run.cancel(err) + } + _ = stream.Close() + <-readDone +} + +func (run *relayRun) write(stream *nativecapture.Stream) error { + waitingRecovery := true + input := relayVideoInput{codec: run.owner.source.codec, onPacketDropped: func() { + waitingRecovery = true + run.owner.source.media.SendPLI(int32(len(run.owner.source.formats)-1), false) + }} + clock := utils.NewWrapAround[uint32, uint64](utils.WrapAroundParams{}) + clock.Update(run.anchorRTP) + for { + select { + case <-run.ctx.Done(): + return context.Cause(run.ctx) + case packet := <-run.packets: + if err := input.push(packet); err != nil { + return err + } + for { + sample, timestamp := input.pop() + if sample == nil { + break + } + frame, err := input.frame(sample.Data) + if err != nil { + return err + } + if waitingRecovery && !frame.KeyFrame { + continue + } + if frame.Width == 0 || frame.Height == 0 { + continue + } + waitingRecovery = false + stamp := clock.Update(timestamp) + if stamp.IsUnhandled || stamp.ExtendedVal < clock.GetExtendedStart() { + continue + } + ticks := stamp.ExtendedVal - clock.GetExtendedStart() + frame.Timestamp = time.Duration(ticks/videoClockRate)*time.Second + time.Duration(ticks%videoClockRate)*time.Second/videoClockRate + frame.Duration = sample.Duration + if frame.Duration <= 0 { + frame.Duration = time.Second / time.Duration(run.plan.profile.Framerate) + } + plan := run.owner.source.relayPlan() + if !plan.wanted || plan.profile != run.plan.profile || plan.format != run.plan.format { + return context.Canceled + } + if run.owner.recovery.Swap(false) { + if err = stream.RequestKeyFrame(-1); err != nil { + return err + } + } + if err = stream.WriteFrame(frame); err != nil { + return err + } + } + } + } +} + +func (run *relayRun) read(stream *nativecapture.Stream) { + for { + frame, err := stream.Read() + if err != nil { + run.cancel(err) + return + } + if run.ctx.Err() != nil { + return + } + source := run.owner.source + switch frame.Kind { + case nativecapture.FrameStatus: + continue + case nativecapture.FrameBegin: + var plan OutputPlan + plan, err = source.beginFrame(frame.Timestamp, run.anchorTime.Add(frame.Timestamp)) + if err == nil { + for slot := range stream.Outputs() { + physical := slot + if slot > 0 { + physical++ + } + if plan.Bitrates[physical] > 0 { + err = stream.SetOutputBitrate(slot, plan.Bitrates[physical]) + } + if err == nil { + err = stream.SetOutputActive(slot, plan.Active[physical]) + } + if err != nil { + break + } + } + } + for _, physical := range plan.RecoveryLayers { + if physical == 1 { + source.RequestRecoveryFrame() + continue + } + slot := physical + if slot > 1 { + slot-- + } + if err == nil { + err = stream.RequestKeyFrame(slot) + } + } + case nativecapture.FrameH264, nativecapture.FrameVP8: + if frame.Layer > source.capacity || (frame.Kind == nativecapture.FrameH264) != (source.codec == "h264") { + err = errors.New("native relay output does not match its source") + break + } + physical := frame.Layer + if physical > 0 { + physical++ + } + err = run.acceptFrame(physical, frame) + case nativecapture.FrameLayerUnavailable: + physical := frame.Layer + if physical > 0 { + physical++ + } + retire, failure := source.markLayerUnavailable(physical, run) + err = failure + if retire != nil { + retire() + } + default: + err = errors.New("native relay output failed") + } + if err != nil { + run.cancel(err) + return + } + } +} + +func (run *relayRun) acceptFrame(slot int, frame nativecapture.Frame) error { + run.owner.mu.Lock() + defer run.owner.mu.Unlock() + source := run.owner.source + source.writeMu.Lock() + defer source.writeMu.Unlock() + source.mu.Lock() + current := run.owner.run == run && run.ctx.Err() == nil && !source.closed && + source.relayProfile == run.plan.profile && source.formats[len(source.formats)-1].Load() == run.plan.format + source.mu.Unlock() + if !current { + return context.Canceled + } + if err := source.setFormat(slot, frame.Width, frame.Height); err != nil { + return err + } + return source.writeVideo(slot, encoded.Frame{ + Data: frame.Data, PTS: frame.Timestamp, Duration: frame.Duration, Recovery: frame.KeyFrame, + }) +} diff --git a/internal/client/mediaedge/relay_fixture_test.go b/internal/client/mediaedge/relay_fixture_test.go new file mode 100644 index 0000000..9046b16 --- /dev/null +++ b/internal/client/mediaedge/relay_fixture_test.go @@ -0,0 +1,483 @@ +package mediaedge + +import ( + "context" + "encoding/hex" + "encoding/json" + "io" + "os" + "testing" + "time" + + "github.com/TNTcraftHIM/Screener/internal/client/nativecapture" + "github.com/livekit/mediatransportutil" + mediacodec "github.com/livekit/mediatransportutil/pkg/codec" + "github.com/pion/rtcp" + "github.com/pion/rtp" + "github.com/pion/rtp/codecs" +) + +// Explicit small native-codec acceptance, never launched by the ordinary suite. +// Forced layer demand tests derivation ownership, not network adaptation. +func TestRelayDerivationFixture(t *testing.T) { + executable, fixture := os.Getenv("SCREENER_NATIVE_CAPTURE"), os.Getenv("SCREENER_ENCODED_FIXTURE") + if executable == "" || fixture == "" { + t.Skip("set SCREENER_NATIVE_CAPTURE and SCREENER_ENCODED_FIXTURE for native relay acceptance") + } + input, err := os.Open(fixture) + if err != nil { + t.Fatal(err) + } + defer input.Close() + var row struct { + Layer int + DataHex string + } + decoder := json.NewDecoder(input) + for index := 0; index < 3; index++ { + if err = decoder.Decode(&row); err != nil { + t.Fatal(err) + } + if row.Layer == 2 { + break + } + } + data, err := hex.DecodeString(row.DataHex) + if err != nil || row.Layer != 2 { + t.Fatal("fixture has no first-frame high recovery output") + } + engine, err := NewEngine(EngineOptions{BindAddress: "127.0.0.1:0", IncludeLoopback: true}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = engine.Close() }) + source, err := engine.NewSource("vp8", 2, 2, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = source.Close() }) + source.relay = &relayDerivation{source: source, options: RelayOptions{ + CaptureProcess: executable, + Capabilities: nativecapture.Capabilities{SoftwareVP8: true, Adapters: []nativecapture.Adapter{{Index: 0}}}, + }} + profile := nativecapture.VideoProfile{Width: 1280, Height: 720, Framerate: 30, Bitrate: 2_000_000, Preference: "balanced"} + if err = source.SetRelayProfile(profile); err != nil { + t.Fatal(err) + } + healthy, receiverA, packetsA := connectedReceiver(t, engine, source, "relay-healthy") + weak, receiverB, packetsB := connectedReceiver(t, engine, source, "relay-weak") + t.Cleanup(func() { _ = receiverA.Close() }) + t.Cleanup(func() { _ = receiverB.Close() }) + packetizer := rtp.NewPacketizer(videoPacketMTU, vp8PayloadType, 1, &codecs.VP8Payloader{EnablePictureID: true}, rtp.NewRandomSequencer(), videoClockRate) + ticker := time.NewTicker(time.Second / 30) + defer ticker.Stop() + var seen [2]mediacodec.VideoSize + var frames [2]int + var packetsSent, bytesSent uint32 + feed := func() { + t.Helper() + for _, packet := range packetizer.Packetize(data, videoClockRate/30) { + if err = source.WriteRTP(packet); err != nil { + t.Fatal(err) + } + packetsSent++ + bytesSent += uint32(len(packet.Payload)) + if packet.Marker { + _ = source.SenderReport(&rtcp.SenderReport{SSRC: 1, RTPTime: packet.Timestamp, + NTPTime: uint64(mediatransportutil.ToNtpTime(time.Now())), PacketCount: packetsSent, OctetCount: bytesSent}) + } + } + <-ticker.C + for child, channel := range []<-chan *rtp.Packet{packetsA, packetsB} { + drain: + for { + select { + case packet := <-channel: + var header mediacodec.VP8 + if header.Unmarshal(packet.Payload) == nil { + size := mediacodec.ExtractVP8VideoSize(&header, packet.Payload) + if size.Width > 0 { + seen[child] = size + } + } + if len(packet.Payload) > 0 && packet.Marker { + frames[child]++ + } + default: + break drain + } + } + } + } + for frame := 0; frame < 60; frame++ { + feed() + } + if frames[0] == 0 || frames[1] == 0 { + t.Fatalf("raw warmup did not deliver both children: %v", frames) + } + source.relay.mu.Lock() + startedEarly := source.relay.run != nil + source.relay.mu.Unlock() + if startedEarly { + t.Fatal("healthy raw forwarding unnecessarily started a decoder") + } + if err = weak.SetTargetLayer(0); err != nil { + t.Fatal(err) + } + // Allow the framework's existing four-second screen-share bitrate window. + for frame := 0; frame < 240 && seen[1].Width != 320; frame++ { + weak.transport.Output.SetBudget(300_000) + feed() + } + if seen[0].Width != 640 || seen[1].Width != 320 || healthy.transport.Output.State().Current != 1 { + source.relay.mu.Lock() + if source.relay.run != nil { + t.Logf("derived process cause=%v pending=%d", context.Cause(source.relay.run.ctx), len(source.relay.run.packets)) + } + source.relay.mu.Unlock() + _, observed := weak.transport.TargetBitrate() + t.Logf("relay plan=%+v feedback=%v ceilings=%v", source.relayPlan(), observed, source.outputBitrates) + available, rates := source.media.GetLayeredBitrate() + t.Logf("actual lower format=%x available=%v rates=%v", source.formats[0].Load(), available, rates) + for layer, buffer := range source.media.GetAllBuffers() { + if buffer == nil { + continue + } + if stats := buffer.GetStats(); stats != nil { + t.Logf("layer=%d frames=%d packets=%d bytes=%d", layer, stats.Frames, stats.Packets, stats.Bytes) + } + } + t.Fatalf("derived low and preserved raw high not delivered: sizes=%+v frames=%v low=%+v", seen, frames, weak.transport.Output.State()) + } + source.relay.mu.Lock() + run := source.relay.run + source.relay.mu.Unlock() + if run == nil { + t.Fatal("no owned shared decoder process") + } + beforeChange := frames + profile.Preference = "maintain-resolution" + if err = source.SetRelayProfile(profile); err != nil { + t.Fatal(err) + } + select { + case <-run.done: + case <-time.After(2 * time.Second): + t.Fatal("old relay settings kept a stale decoder process alive") + } + for frame := 0; frame < 90; frame++ { + weak.transport.Output.SetBudget(300_000) + feed() + if frames[1] > beforeChange[1]+2 { + break + } + } + source.relay.mu.Lock() + replacement := source.relay.run + source.relay.mu.Unlock() + if replacement == nil || replacement == run || frames[0] <= beforeChange[0] || frames[1] <= beforeChange[1] { + t.Fatal("live relay settings stopped healthy forwarding or failed to replace the derived output") + } + run = replacement + if err = healthy.Close(); err != nil { + t.Fatal(err) + } + if err = receiverA.Close(); err != nil { + t.Fatal(err) + } + secondLow, secondReceiver, secondPackets := connectedReceiver(t, engine, source, "relay-second-low") + t.Cleanup(func() { _ = secondReceiver.Close() }) + packetsA = secondPackets + seen[0], frames[0] = mediacodec.VideoSize{}, 0 + if err = secondLow.SetTargetLayer(0); err != nil { + t.Fatal(err) + } + beforeShared := frames + checkSharedRun := func() { + t.Helper() + source.relay.mu.Lock() + same := source.relay.run == run + source.relay.mu.Unlock() + select { + case <-run.done: + t.Fatal("a compatible consumer retired the shared derivation") + default: + } + if !same { + t.Fatal("compatible consumers created a replacement derivation") + } + } + for frame := 0; frame < 120; frame++ { + weak.transport.Output.SetBudget(300_000) + secondLow.transport.Output.SetBudget(300_000) + feed() + checkSharedRun() + _, observed := secondLow.transport.TargetBitrate() + if observed && seen[0].Width == 320 && frames[0] >= 3 && frames[1] >= beforeShared[1]+3 && + weak.transport.CurrentSource() == secondLow.transport.CurrentSource() { + break + } + } + _, secondObserved := secondLow.transport.TargetBitrate() + if !secondObserved || seen[0].Width != 320 || frames[0] < 3 || frames[1] < beforeShared[1]+3 || + weak.transport.CurrentSource() != secondLow.transport.CurrentSource() { + t.Fatalf("compatible low consumers did not share delivery: sizes=%+v frames=%v", seen, frames) + } + checkMembership := func(split bool) bool { + source.writeMu.Lock() + defer source.writeMu.Unlock() + weakGroup, secondGroup := source.memberships[weak.transport], source.memberships[secondLow.transport] + if weakGroup == nil || secondGroup == nil || (weakGroup != secondGroup) != split || + weak.transport.CurrentSource() != weakGroup.media.Source || + secondLow.transport.CurrentSource() != secondGroup.media.Source { + return false + } + weakBudget := uint32(300_000) + if split { + weakBudget = 80_000 + } + return weakGroup.budget == weakBudget && secondGroup.budget == 300_000 && + weak.transport.Output.State().Current == 0 && secondLow.transport.Output.State().Current == 0 + } + for _, split := range []bool{true, false} { + before := frames + budget := int64(300_000) + if split { + budget = 80_000 + } + for frame := 0; frame < 120; frame++ { + weak.transport.Output.SetBudget(budget) + secondLow.transport.Output.SetBudget(300_000) + feed() + checkSharedRun() + if checkMembership(split) && frames[0] >= before[0]+3 && frames[1] >= before[1]+3 { + break + } + } + if !checkMembership(split) || frames[0] < before[0]+3 || frames[1] < before[1]+3 { + t.Fatalf("budget membership transition failed: split=%v frames=%v->%v weak=%+v second=%+v", + split, before, frames, weak.transport.Output.State(), secondLow.transport.Output.State()) + } + t.Logf("shared decoder budget transition: split=%v weakBudget=%d secondBudget=300000 frames=%v->%v", + split, budget, before, frames) + } + if err = weak.Close(); err != nil { + t.Fatal(err) + } + beforeRetire := frames[0] + for frame := 0; frame < 30 && frames[0] < beforeRetire+3; frame++ { + secondLow.transport.Output.SetBudget(300_000) + feed() + checkSharedRun() + } + if frames[0] < beforeRetire+3 { + t.Fatal("retiring one consumer stopped the other consumer's shared output") + } + if err = secondLow.Close(); err != nil { + t.Fatal(err) + } + // Only the last lower-output consumer retires the derivation. + for frame := 0; frame < 3; frame++ { + feed() + } + select { + case <-run.done: + case <-time.After(2 * time.Second): + t.Fatal("unused native relay process did not retire") + } + t.Log("640x360 raw forwarding; lower consumers shared, split at 80/300 kbps, rejoined at 300/300 kbps and independently retired with one decoder process") +} + +// Forced direct-child budgets exercise real per-group WebRTC spatial adaptation; +// they do not simulate transport loss or prove bandwidth-estimator behavior. +func TestRelayIndependentSpatialFixture(t *testing.T) { + executable, fixture := os.Getenv("SCREENER_NATIVE_CAPTURE"), os.Getenv("SCREENER_GROUP_MOTION_INPUT") + if executable == "" || fixture == "" { + t.Skip("set SCREENER_NATIVE_CAPTURE and SCREENER_GROUP_MOTION_INPUT") + } + check := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + input, err := os.Open(fixture) + check(err) + defer input.Close() + var motion [][]byte + decoder := json.NewDecoder(io.LimitReader(input, 16*1024*1024)) + for { + var row struct { + Width, Height uint32 + Recovery bool + DataHex string + } + err := decoder.Decode(&row) + if err == io.EOF { + break + } + check(err) + if len(motion) >= 60 || row.Width != 1280 || row.Height != 720 || len(motion) == 0 && !row.Recovery { + t.Fatal("expected bounded 1280x720 motion input beginning with recovery") + } + data, err := hex.DecodeString(row.DataHex) + check(err) + motion = append(motion, data) + } + if len(motion) != 60 { + t.Fatal("motion fixture must contain 60 decoder-safe frames") + } + engine, err := NewEngine(EngineOptions{BindAddress: "127.0.0.1:0", IncludeLoopback: true}) + check(err) + defer engine.Close() + source, err := engine.NewSource("vp8", 2, 2, nil) + check(err) + defer source.Close() + source.relay = &relayDerivation{source: source, options: RelayOptions{ + CaptureProcess: executable, + Capabilities: nativecapture.Capabilities{SoftwareVP8: true, Adapters: []nativecapture.Adapter{{Index: 0}}}, + }} + check(source.SetRelayProfile(nativecapture.VideoProfile{ + Width: 1920, Height: 1080, Framerate: 30, Bitrate: 5_000_000, Preference: "maintain-framerate", + })) + a, receiverA, packetsA := connectedReceiver(t, engine, source, "spatial-a") + b, receiverB, packetsB := connectedReceiver(t, engine, source, "spatial-b") + defer receiverA.Close() + defer receiverB.Close() + packetizer := rtp.NewPacketizer(videoPacketMTU, vp8PayloadType, 1, &codecs.VP8Payloader{EnablePictureID: true}, rtp.NewRandomSequencer(), videoClockRate) + ticker := time.NewTicker(time.Second / 30) + defer ticker.Stop() + ctx, cancel := context.WithTimeout(t.Context(), 35*time.Second) + defer cancel() + type receivedFrame struct { + Index int + Width, Height uint32 + Recovery bool + Data []byte + PTS time.Duration + } + var output [2][]receivedFrame + var seen [2]mediacodec.VideoSize + var frames [2]int + var anchor [2]uint32 + var waiting = [2]bool{true, true} + var assembled [2]relayVideoInput + for child := range assembled { + assembled[child] = relayVideoInput{codec: "vp8", onPacketDropped: func() { waiting[child] = true }} + } + var packetsSent, bytesSent uint32 + index := 0 + record := false + feed := func() { + t.Helper() + for _, packet := range packetizer.Packetize(motion[index%len(motion)], videoClockRate/30) { + check(source.WriteRTP(packet)) + packetsSent++ + bytesSent += uint32(len(packet.Payload)) + if packet.Marker { + check(source.SenderReport(&rtcp.SenderReport{SSRC: 1, RTPTime: packet.Timestamp, + NTPTime: uint64(mediatransportutil.ToNtpTime(time.Now())), PacketCount: packetsSent, OctetCount: bytesSent})) + } + } + index++ + select { + case <-ticker.C: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + for child, channel := range []<-chan *rtp.Packet{packetsA, packetsB} { + drain: + for { + select { + case packet := <-channel: + check(assembled[child].push(packet)) + for { + sample, timestamp := assembled[child].pop() + if sample == nil { + break + } + frame, err := assembled[child].frame(sample.Data) + check(err) + seen[child] = mediacodec.VideoSize{Width: frame.Width, Height: frame.Height} + frames[child]++ + if !record || waiting[child] && !frame.KeyFrame { + continue + } + if len(output[child]) == 0 { + anchor[child] = timestamp + } + waiting[child] = false + output[child] = append(output[child], receivedFrame{Index: len(output[child]), + Width: frame.Width, Height: frame.Height, Recovery: frame.KeyFrame, Data: frame.Data, + PTS: time.Duration(uint32(timestamp-anchor[child])) * time.Second / videoClockRate}) + } + default: + break drain + } + } + } + } + for range 60 { + feed() + } + check(a.SetTargetLayer(0)) + check(b.SetTargetLayer(0)) + for frame := 0; frame < 240; frame++ { + a.transport.Output.SetBudget(1_250_000) + b.transport.Output.SetBudget(1_250_000) + feed() + if seen[0].Width == 640 && seen[1].Width == 640 && a.transport.Output.State().Current == 0 && b.transport.Output.State().Current == 0 { + break + } + } + if seen[0].Width != 640 || seen[1].Width != 640 { + t.Fatalf("shared derived warmup failed: sizes=%+v frames=%v", seen, frames) + } + source.relay.mu.Lock() + run := source.relay.run + source.relay.mu.Unlock() + if run == nil { + t.Fatal("motion derivation did not start") + } + before := frames + record = true + for range 600 { + a.transport.Output.SetBudget(80_000) + b.transport.Output.SetBudget(1_250_000) + feed() + source.relay.mu.Lock() + same := source.relay.run == run + source.relay.mu.Unlock() + if !same { + t.Fatal("independent budgets replaced the shared decoder process") + } + select { + case <-run.done: + t.Fatalf("shared motion derivation failed: %v", context.Cause(run.ctx)) + default: + } + } + if path := os.Getenv("SCREENER_GROUP_OUTPUT"); path != "" { + file, err := os.Create(path) + check(err) + check(json.NewEncoder(file).Encode(output)) + check(file.Close()) + } + source.writeMu.Lock() + ga, gb := source.memberships[a.transport], source.memberships[b.transport] + independent := ga != nil && gb != nil && ga != gb && ga.budget == 80_000 && gb.budget == 1_250_000 && + a.transport.CurrentSource() == ga.media.Source && b.transport.CurrentSource() == gb.media.Source + source.writeMu.Unlock() + if !independent || seen[0].Width != 320 || seen[0].Height != 180 || seen[1].Width != 640 || seen[1].Height != 360 || + frames[0] <= before[0]+30 || frames[1] <= before[1]+30 || a.transport.Output.State().Current != 0 || b.transport.Output.State().Current != 0 { + t.Fatalf("independent spatial adaptation missing: independent=%v sizes=%+v frames=%v->%v", independent, seen, before, frames) + } + check(a.Close()) + check(b.Close()) + select { + case <-run.done: + case <-time.After(2 * time.Second): + t.Fatal("last spatial consumer did not retire the decoder process") + } + t.Logf("one decoder, two budget-owned encoders: A=80kbps 320x180 B=1250kbps 640x360 frames=%v->%v exported=%d/%d", before, frames, len(output[0]), len(output[1])) +} diff --git a/internal/client/mediaedge/relay_input.go b/internal/client/mediaedge/relay_input.go new file mode 100644 index 0000000..f715a22 --- /dev/null +++ b/internal/client/mediaedge/relay_input.go @@ -0,0 +1,128 @@ +package mediaedge + +import ( + "bytes" + "errors" + "io" + + "github.com/TNTcraftHIM/Screener/internal/client/nativecapture" + mediacodec "github.com/livekit/mediatransportutil/pkg/codec" + "github.com/livekit/server-sdk-go/v2/pkg/samplebuilder" + "github.com/pion/rtp" + "github.com/pion/rtp/codecs" + "github.com/pion/webrtc/v4/pkg/media" + "github.com/pion/webrtc/v4/pkg/media/h264reader" +) + +// Decoder configuration belongs to this received stream, not a child encoder. +type relayVideoInput struct { + codec string + sps []byte + pps []byte + width uint32 + height uint32 + builder *samplebuilder.SampleBuilder + onPacketDropped func() + recoveryTimestamp uint32 + recoverySequence uint16 + hasRecovery bool +} + +func (input *relayVideoInput) push(packet *rtp.Packet) error { + if input.hasRecovery && int32(packet.Timestamp-input.recoveryTimestamp) < 0 { + return nil + } + recovery := mediacodec.IsKeyFrame(input.codec, packet.Payload) + if recovery && input.hasRecovery && packet.Timestamp == input.recoveryTimestamp && packet.SequenceNumber == input.recoverySequence { + return nil + } + if input.builder == nil || recovery && (!input.hasRecovery || packet.Timestamp != input.recoveryTimestamp) { + if input.builder != nil { + for { + sample, _ := input.builder.ForcePopWithTimestamp() + if sample == nil { + break + } + // Keep completed H264 configuration, never replay expired video. + if input.codec == "h264" { + if _, err := input.frame(sample.Data); err != nil { + return err + } + } + } + } + var depacketizer rtp.Depacketizer = &codecs.H264Packet{} + if input.codec == "vp8" { + depacketizer = &codecs.VP8Packet{} + } + input.builder = samplebuilder.New(relayPacketLimit, depacketizer, videoClockRate, + samplebuilder.WithPacketDroppedHandler(func() { + if input.onPacketDropped != nil { + input.onPacketDropped() + } + })) + } + if recovery { + input.recoveryTimestamp, input.hasRecovery = packet.Timestamp, true + input.recoverySequence = packet.SequenceNumber + } + input.builder.Push(packet) + return nil +} + +func (input *relayVideoInput) pop() (*media.Sample, uint32) { + return input.builder.PopWithTimestamp() +} + +func (input *relayVideoInput) frame(data []byte) (nativecapture.Frame, error) { + frame := nativecapture.Frame{Data: data} + if input.codec == "vp8" { + frame.Kind = nativecapture.FrameVP8 + frame.KeyFrame = len(data) >= 10 && data[0]&1 == 0 + size := mediacodec.ExtractVP8VideoSize(&mediacodec.VP8{IsKeyFrame: frame.KeyFrame}, data) + if size.Width > 0 && size.Height > 0 { + input.width, input.height = size.Width, size.Height + } + } else { + frame.Kind = nativecapture.FrameH264 + reader, err := h264reader.NewReader(bytes.NewReader(data)) + if err != nil { + return frame, err + } + for { + nal, err := reader.NextNAL() + if errors.Is(err, io.EOF) { + break + } + if err != nil { + return frame, err + } + switch nal.UnitType { + case h264reader.NalUnitTypeSPS: + input.sps = append(input.sps[:0], nal.Data...) + size := mediacodec.ExtractH264VideoSize(nal.Data) + if size.Width > 0 && size.Height > 0 { + input.width, input.height = size.Width, size.Height + } + case h264reader.NalUnitTypePPS: + input.pps = append(input.pps[:0], nal.Data...) + case h264reader.NalUnitTypeCodedSliceIdr: + frame.KeyFrame = true + } + } + if frame.KeyFrame { + frame.KeyFrame = len(input.sps) > 0 && len(input.pps) > 0 + if frame.KeyFrame { + // Browser senders can put parameter sets in an earlier access unit. + prefix := make([]byte, 0, len(input.sps)+len(input.pps)+len(data)+8) + for _, nal := range [][]byte{input.sps, input.pps} { + prefix = append(prefix, 0, 0, 0, 1) + prefix = append(prefix, nal...) + } + frame.Data = append(prefix, data...) + } + } + } + frame.Width, frame.Height = input.width, input.height + return frame, nil +} diff --git a/internal/client/mediaedge/relay_input_test.go b/internal/client/mediaedge/relay_input_test.go new file mode 100644 index 0000000..07211df --- /dev/null +++ b/internal/client/mediaedge/relay_input_test.go @@ -0,0 +1,213 @@ +package mediaedge + +import ( + "bytes" + "encoding/hex" + "sync/atomic" + "testing" + "time" + + "github.com/TNTcraftHIM/Screener/internal/media/encoded" + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/pion/rtp" + "github.com/pion/rtp/codecs" + "github.com/pion/webrtc/v4" +) + +func TestRelayInputKeepsRecoveryAndEmitsACompleteQuietFrame(t *testing.T) { + // A single complete access unit must not need a subsequent frame or timer. + key, err := hex.DecodeString("1000009d012aa0005a00") + if err != nil { + t.Fatal(err) + } + input := relayVideoInput{codec: "vp8"} + if err = input.push(&rtp.Packet{Header: rtp.Header{Version: 2, SequenceNumber: 10, Timestamp: 0xfffffff0, Marker: true}, + Payload: append([]byte{0x10}, key...)}); err != nil { + t.Fatal(err) + } + sample, timestamp := input.pop() + if sample == nil || timestamp != 0xfffffff0 || !bytes.Equal(sample.Data, key) { + t.Fatal("complete quiet frame waited for future input") + } + frame, err := input.frame(sample.Data) + if err != nil || !frame.KeyFrame || frame.Width != 160 || frame.Height != 90 { + t.Fatalf("VP8 recovery metadata = %+v, %v", frame, err) + } + frame, err = input.frame([]byte{1, 0, 0, 1}) + if err != nil || frame.KeyFrame || frame.Width != 160 || frame.Height != 90 { + t.Fatal("delta frame lost the stream's dimensions") + } + + input = relayVideoInput{codec: "h264"} + configuration, _ := hex.DecodeString("000000016742c01eda0280b7fe5c050505020000000168ce06e2") + if _, err = input.frame(configuration); err != nil { + t.Fatal(err) + } + frame, err = input.frame([]byte{0, 0, 0, 1, 0x65, 0x88, 0x84}) + if err != nil || !frame.KeyFrame || !bytes.Contains(frame.Data, configuration[:4+len(input.sps)]) || + !bytes.Contains(frame.Data, input.pps) { + t.Fatal("H264 IDR lost preceding decoder configuration") + } +} + +func TestLargeRecoveryUsesTheDefaultPacketWindow(t *testing.T) { + check := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + // Standard user-data SEI plus the library's valid H264 keyframe. The SEI + // makes this AU exceed 500 RTP packets without a codec/GPU test dependency. + var data []byte + for _, nal := range sfu.H264KeyFrame2x2[:2] { + data = append(data, 0, 0, 0, 1) + data = append(data, nal...) + } + data = append(data, 0, 0, 0, 1, 0x06, 5) + const userBytes = 1024 * 1024 + for size := userBytes; size >= 255; size -= 255 { + data = append(data, 255) + } + data = append(data, userBytes%255) + data = append(data, bytes.Repeat([]byte{0x55}, userBytes)...) + data = append(data, 0x80, 0, 0, 0, 1) + data = append(data, sfu.H264KeyFrame2x2IDR...) + packetizer := encoded.NewPacketizer(&codecs.H264Payloader{}, h264PayloadType, videoPacketMTU) + packets, err := packetizer.Packetize(data, 0, time.Second) + check(err) + if len(packets) <= 500 || len(packets) > encoded.MaxPacketWindow { + t.Fatal("fixture does not exercise the packet window boundary") + } + engine, err := NewEngine(EngineOptions{BindAddress: "127.0.0.1:0", IncludeLoopback: true, InitialBitrate: 12_000_000}) + check(err) + defer engine.Close() + source, err := engine.NewSource("h264", 1, 1, nil) + check(err) + defer source.Close() + check(source.SetFormat(0, 2, 2)) + check(source.ConfigureOutputs([]uint32{12_000_000})) + edge, err := engine.NewEdge(source, EdgeOptions{ConnectionID: "large-recovery"}) + check(err) + defer edge.Close() + receiver := newReceiver(t) + defer receiver.Close() + var complete atomic.Bool + receiver.OnTrack(func(track *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { + input := relayVideoInput{codec: "h264"} + var parts []byte + var previous uint32 + for { + packet, _, err := track.ReadRTP() + if err != nil { + return + } + if err = input.push(packet); err != nil { + t.Error(err) + return + } + for { + sample, timestamp := input.pop() + if sample == nil { + break + } + if timestamp != previous { + parts = parts[:0] + previous = timestamp + } + parts = append(parts, sample.Data...) + if bytes.Equal(parts, data) { + complete.Store(true) + } + } + } + }) + connectEdgeToReceiver(t, edge, receiver) + for attempt := 1; attempt <= 4 && !complete.Load(); attempt++ { + check(writeSourceFrame(source, data, time.Duration(attempt)*time.Second, time.Second)) + time.Sleep(time.Second) + } + if !complete.Load() || edge.transport.Egress().DroppedPackets != 0 { + t.Fatalf("large recovery did not survive source, pacer and assembly: packets=%d complete=%v egress=%+v", len(packets), complete.Load(), edge.transport.Egress()) + } +} + +func TestRelayAssemblyRecoversAtNewIndependentFrame(t *testing.T) { + key, _ := hex.DecodeString("1000009d012aa0005a00") + sps, _ := hex.DecodeString("6742c01eda0280b7fe5c0505050200") + pps := []byte{0x68, 0xce, 0x06, 0xe2} + idr := []byte{0x65, 0x88, 0x84} + stap := []byte{0x78} + for _, nal := range [][]byte{sps, pps, idr} { + stap = append(stap, byte(len(nal)>>8), byte(len(nal))) + stap = append(stap, nal...) + } + for _, check := range []struct { + name, codec string + recovery [][]byte + }{ + {"vp8", "vp8", [][]byte{append([]byte{0x10}, key...), {0x00, 0x01}}}, + {"h264-nals", "h264", [][]byte{sps, pps, idr}}, + {"h264-stap", "h264", [][]byte{stap}}, + {"h264-fu", "h264", [][]byte{append([]byte{0x7c, 0x87}, sps[1:5]...), append([]byte{0x7c, 0x47}, sps[5:]...), pps, idr}}, + } { + t.Run(check.name, func(t *testing.T) { + input := relayVideoInput{codec: check.codec} + old := uint32(0xffffe000) + fresh := uint32(0x1000) + push := func(sequence uint16, timestamp uint32, marker bool, payload []byte) { + t.Helper() + if err := input.push(&rtp.Packet{Header: rtp.Header{Version: 2, SequenceNumber: sequence, Timestamp: timestamp, Marker: marker}, Payload: payload}); err != nil { + t.Fatal(err) + } + } + for index, payload := range check.recovery { + push(uint16(10+index), old, index+1 == len(check.recovery), payload) + } + if sample, _ := input.pop(); sample == nil { + t.Fatal("initial complete recovery did not assemble") + } + for sample, _ := input.pop(); sample != nil; sample, _ = input.pop() { + } + // A missing sequence and partial frame must not block later independent recovery. + partial := []byte{0x10, 0x01, 0x00} + if check.codec == "h264" { + partial = []byte{0x7c, 0x81, 0x01} + } + push(30, old+3000, false, partial) + if sample, _ := input.pop(); sample != nil { + t.Fatal("incomplete frame unexpectedly assembled") + } + for index, payload := range check.recovery { + push(uint16(40+index), fresh, index+1 == len(check.recovery), payload) + if index == 0 { + push(40, fresh, index+1 == len(check.recovery), payload) + push(10, old, false, payload) + } + } + sample, timestamp := input.pop() + if sample == nil || timestamp != fresh { + t.Fatalf("new recovery remained blocked: sample=%v timestamp=%x recovery=%x", sample != nil, timestamp, input.recoveryTimestamp) + } + recovered := false + for ; sample != nil; sample, timestamp = input.pop() { + if timestamp != fresh { + t.Fatal("old frame was replayed after recovery") + } + frame, err := input.frame(sample.Data) + if err != nil { + t.Fatal(err) + } + if frame.KeyFrame { + if frame.Width == 0 || frame.Height == 0 { + t.Fatal("recovered dimensions missing") + } + recovered = true + } + } + if !recovered { + t.Fatal("complete independent recovery was not assembled") + } + }) + } +} diff --git a/internal/client/mediaedge/relay_network_fixture_test.go b/internal/client/mediaedge/relay_network_fixture_test.go new file mode 100644 index 0000000..3cf4080 --- /dev/null +++ b/internal/client/mediaedge/relay_network_fixture_test.go @@ -0,0 +1,353 @@ +package mediaedge + +import ( + "bytes" + "context" + "encoding/hex" + "encoding/json" + "io" + "os" + "sync" + "testing" + "time" + + "github.com/TNTcraftHIM/Screener/internal/client/nativecapture" + "github.com/livekit/mediatransportutil" + "github.com/pion/ice/v4" + "github.com/pion/interceptor" + "github.com/pion/logging" + "github.com/pion/rtcp" + "github.com/pion/rtp" + "github.com/pion/rtp/codecs" + "github.com/pion/transport/v4/vnet" + "github.com/pion/webrtc/v4" +) + +// One real VP8 derived process, no desktop capture. The link changes capacity; +// production BWE, output admission and codec controls supply every adaptation. +func TestRelayNetworkVP8Fixture(t *testing.T) { + if os.Getenv("SCREENER_RELAY_NETWORK") == "" { + t.Skip("set SCREENER_RELAY_NETWORK=1, SCREENER_NATIVE_CAPTURE and SCREENER_ENCODED_FIXTURE") + } + check := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + input, err := os.Open(os.Getenv("SCREENER_ENCODED_FIXTURE")) + check(err) + defer input.Close() + var motion [][]byte + decoder := json.NewDecoder(input) + for records := 0; ; records++ { + var row struct { + FrameIndex, Layer int + DataHex string + } + if err = decoder.Decode(&row); err == io.EOF { + break + } + check(err) + if records >= 96 || row.FrameIndex < 0 || row.FrameIndex >= 40 || row.Layer < 0 || row.Layer > 2 || len(row.DataHex) > 8*1024*1024 { + t.Fatal("encoded fixture exceeded its bounds") + } + if row.Layer == 2 && row.FrameIndex < 16 { + data, decodeErr := hex.DecodeString(row.DataHex) + check(decodeErr) + motion = append(motion, data) + } + } + if len(motion) != 16 { + t.Fatal("fixture needs its complete first sixteen high outputs") + } + ctx, cancel := context.WithTimeout(t.Context(), 40*time.Second) + defer cancel() + engine, err := NewEngine(EngineOptions{BindAddress: "127.0.0.1:0", InitialBitrate: 1_200_000}) + check(err) + defer engine.Close() + source, err := engine.NewSource("vp8", 2, 2, nil) + check(err) + defer source.Close() + source.relay = &relayDerivation{source: source, options: RelayOptions{ + CaptureProcess: os.Getenv("SCREENER_NATIVE_CAPTURE"), + Capabilities: nativecapture.Capabilities{SoftwareVP8: true, Adapters: []nativecapture.Adapter{{Index: 0}}}, + }} + check(source.SetRelayProfile(nativecapture.VideoProfile{Width: 1280, Height: 720, + Framerate: 30, Bitrate: 2_000_000, Preference: "balanced"})) + router, err := vnet.NewRouter(&vnet.RouterConfig{CIDR: "10.0.0.0/24", MinDelay: 20 * time.Millisecond, + LoggerFactory: logging.NewDefaultLoggerFactory()}) + check(err) + sendNet, err := vnet.NewNet(&vnet.NetConfig{StaticIPs: []string{"10.0.0.1"}}) + check(err) + check(router.AddNet(sendNet)) + engine.settings = webrtc.SettingEngine{} + engine.settings.SetNet(sendNet) + engine.settings.SetNetworkTypes([]webrtc.NetworkType{webrtc.NetworkTypeUDP4}) + engine.settings.SetICEMulticastDNSMode(ice.MulticastDNSModeDisabled) + link := vnet.NewTBFQueue(6_000_000, 2400, 2400) + type deliveredFrame struct { + Index, Layer, Width, Height int + PTS time.Duration + Recovery bool + Data []byte + ReceivedAt, DeliveryAge time.Duration + } + var receivedMu sync.Mutex + var delivered [2][]deliveredFrame + var lost [2]int + var intervalPacketFrames [2]int + var intervalPacketBytes [2]int + var intervalPacketAge [2]time.Duration + var sourceOrigin time.Time + var inputTimes []time.Time + var children [2]*Edge + var receivers [2]*webrtc.PeerConnection + for child, address := range []string{"10.0.0.2", "10.0.0.3"} { + network, netErr := vnet.NewNet(&vnet.NetConfig{StaticIPs: []string{address}}) + check(netErr) + if child == 1 { + queue, queueErr := vnet.NewQueue(network, link) + check(queueErr) + defer queue.Close() + check(router.AddNet(queue)) + } else { + check(router.AddNet(network)) + } + settings := webrtc.SettingEngine{} + settings.SetNet(network) + settings.SetNetworkTypes([]webrtc.NetworkType{webrtc.NetworkTypeUDP4}) + settings.SetICEMulticastDNSMode(ice.MulticastDNSModeDisabled) + media := &webrtc.MediaEngine{} + check(media.RegisterDefaultCodecs()) + registry := &interceptor.Registry{} + check(webrtc.RegisterDefaultInterceptors(media, registry)) + check(webrtc.ConfigureTWCCSender(media, registry)) + receiver, createErr := webrtc.NewAPI(webrtc.WithMediaEngine(media), webrtc.WithInterceptorRegistry(registry), + webrtc.WithSettingEngine(settings)).NewPeerConnection(webrtc.Configuration{}) + check(createErr) + receivers[child] = receiver + defer receiver.Close() + receiver.OnTrack(func(track *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { + waitingKey, anchored := true, false + var originRTP uint32 + video := relayVideoInput{codec: "vp8", onPacketDropped: func() { + waitingKey = true + receivedMu.Lock() + lost[child]++ + receivedMu.Unlock() + _ = receiver.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: uint32(track.SSRC())}}) + }} + for { + packet, _, readErr := track.ReadRTP() + if readErr != nil { + return + } + receivedMu.Lock() + intervalPacketBytes[child] += len(packet.Payload) + if len(packet.Payload) > 0 && packet.Marker { + intervalPacketFrames[child]++ + } + if anchored && len(packet.Payload) > 0 && !packet.Padding { + inputIndex := int((uint64(packet.Timestamp-originRTP) + 1500) / 3000) + if inputIndex < len(inputTimes) { + intervalPacketAge[child] = max(intervalPacketAge[child], time.Since(inputTimes[inputIndex])) + } + } + receivedMu.Unlock() + if readErr = video.push(packet); readErr != nil { + t.Error(readErr) + return + } + for { + sample, timestamp := video.pop() + if sample == nil { + break + } + frame, frameErr := video.frame(sample.Data) + if frameErr != nil || waitingKey && !frame.KeyFrame { + continue + } + waitingKey = false + if !anchored { + for index, original := range motion { + if bytes.Equal(original, sample.Data) { + originRTP = timestamp - uint32(index*3000) + anchored = true + break + } + } + if !anchored { + continue + } + } + pts := time.Duration(uint32(timestamp-originRTP)) * time.Second / videoClockRate + now := time.Now() + receivedMu.Lock() + inputIndex := int((uint64(timestamp-originRTP) + 1500) / 3000) + age := time.Duration(-1) + if inputIndex < len(inputTimes) { + age = now.Sub(inputTimes[inputIndex]) + } + if len(delivered[child]) < 1200 { + delivered[child] = append(delivered[child], deliveredFrame{Index: len(delivered[child]), + Layer: int(frame.Width / 640), Width: int(frame.Width), Height: int(frame.Height), PTS: pts, + Recovery: frame.KeyFrame, Data: sample.Data, ReceivedAt: now.Sub(sourceOrigin), DeliveryAge: age}) + } + receivedMu.Unlock() + } + } + }) + children[child], err = engine.NewEdge(source, EdgeOptions{ConnectionID: address}) + check(err) + } + check(router.Start()) + defer router.Stop() + for index, child := range children { + connectEdgeToReceiver(t, child, receivers[index]) + } + packetizer := rtp.NewPacketizer(videoPacketMTU, vp8PayloadType, 1, &codecs.VP8Payloader{EnablePictureID: true}, rtp.NewRandomSequencer(), videoClockRate) + period := time.Second / 30 + ticker := time.NewTicker(period) + defer ticker.Stop() + receivedMu.Lock() + sourceOrigin = time.Now() + receivedMu.Unlock() + lastReport := sourceOrigin + var inputPackets, inputBytes uint32 + type observation struct { + Second float64 + Capacity, CodecBudget uint32 + VideoBudget int64 + Target, Current int32 + DerivedFrames, DerivedBytes uint64 + ReceivedFrames [2]int + ReceivedBytes [2]int + MaximumAge [2]time.Duration + PacketFrames [2]int + PacketBytes [2]int + MaximumPacketAge [2]time.Duration + } + var observations []observation + var previousFrames [2]int + var process *relayRun + var processStarts []float64 + var constrained, recovered bool + for frameIndex := 0; ; frameIndex++ { + select { + case <-ticker.C: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + now := time.Now() + elapsed := now.Sub(sourceOrigin) + if elapsed >= 30*time.Second { + break + } + capacity := uint32(6_000_000) + if elapsed >= 5*time.Second && elapsed < 18*time.Second { + capacity = 120_000 + } + link.SetRate(int(capacity)) + receivedMu.Lock() + inputTimes = append(inputTimes, now) + receivedMu.Unlock() + for _, packet := range packetizer.Packetize(motion[frameIndex%len(motion)], 3000) { + check(source.WriteRTP(packet)) + inputPackets++ + inputBytes += uint32(len(packet.Payload)) + if packet.Marker { + check(source.SenderReport(&rtcp.SenderReport{SSRC: 1, RTPTime: packet.Timestamp, + NTPTime: uint64(mediatransportutil.ToNtpTime(now)), PacketCount: inputPackets, OctetCount: inputBytes})) + } + } + source.relay.mu.Lock() + if run := source.relay.run; run != nil { + if process != nil && process != run { + select { + case <-process.done: + default: + t.Error("derived codec processes overlapped") + } + } + if process != run { + processStarts = append(processStarts, elapsed.Seconds()) + } + process = run + } + source.relay.mu.Unlock() + if now.Sub(lastReport) >= time.Second { + state, plan := children[1].transport.Output.State(), source.relayPlan() + physical := 0 + source.writeMu.Lock() + currentSource := children[1].transport.CurrentSource() + if group := source.groupForMedia(currentSource); group != nil { + physical = group.slot + } + source.writeMu.Unlock() + row := observation{Second: elapsed.Seconds(), Capacity: capacity, CodecBudget: plan.controls.Bitrates[physical], + VideoBudget: state.VideoBudget, Target: state.Target, Current: state.Current} + if buffer := currentSource.GetAllBuffers()[0]; buffer != nil { + if stats := buffer.GetStats(); stats != nil { + row.DerivedFrames, row.DerivedBytes = uint64(stats.Frames), stats.Bytes + } + } + receivedMu.Lock() + row.PacketFrames, row.PacketBytes, row.MaximumPacketAge = intervalPacketFrames, intervalPacketBytes, intervalPacketAge + intervalPacketFrames, intervalPacketBytes, intervalPacketAge = [2]int{}, [2]int{}, [2]time.Duration{} + for child := range delivered { + for _, output := range delivered[child][previousFrames[child]:] { + row.ReceivedFrames[child]++ + row.ReceivedBytes[child] += len(output.Data) + row.MaximumAge[child] = max(row.MaximumAge[child], output.DeliveryAge) + if child == 1 && capacity == 120_000 && output.Width == 320 && state.VideoBudget < int64(plan.output.Bitrate) { + constrained = true + } + if child == 1 && elapsed > 18*time.Second && output.Width == 640 && output.PTS > 18*time.Second { + recovered = true + } + if child == 0 && output.Width != 640 { + t.Error("healthy child lost original representation") + } + } + previousFrames[child] = len(delivered[child]) + } + receivedMu.Unlock() + observations = append(observations, row) + t.Logf("%+.1fs capacity=%d budget=%d codec=%d output=%d/%d assembled=%v packetFrames=%v packetBytes=%v maxPacketAge=%v maxAUAge=%v", + row.Second, row.Capacity, row.VideoBudget, row.CodecBudget, row.Target, row.Current, row.ReceivedFrames, row.PacketFrames, row.PacketBytes, row.MaximumPacketAge, row.MaximumAge) + lastReport = now + } + } + check(source.Close()) + for _, receiver := range receivers { + check(receiver.Close()) + } + receivedMu.Lock() + defer receivedMu.Unlock() + // The last reporting tick precedes shutdown; include its unreported tail + // with the same original-size and post-release capture-time requirements. + for _, output := range delivered[1][previousFrames[1]:] { + if output.ReceivedAt > 18*time.Second && output.Width == 640 && output.PTS > 18*time.Second { + recovered = true + } + } + outputPath := os.Getenv("SCREENER_ENCODED_OUTPUT") + if outputPath == "" { + t.Fatal("SCREENER_ENCODED_OUTPUT must name the ignored received-frame artifact") + } + data, err := json.Marshal(delivered) + check(err) + check(os.WriteFile(outputPath, data, 0600)) + data, err = json.MarshalIndent(struct { + Observations []observation + DroppedFrames [2]int + ProcessStarts []float64 + Constrained, Recovered bool + }{observations, lost, processStarts, constrained, recovered}, "", " ") + check(err) + check(os.WriteFile(outputPath+".observations.json", data, 0600)) + if !constrained || !recovered || process == nil || lost[0] != 0 { + t.Fatalf("actual codec/network result: constrained=%v recovered=%v process=%v loss=%v", constrained, recovered, process != nil, lost) + } +} diff --git a/internal/client/mediaedge/source.go b/internal/client/mediaedge/source.go index aac9e1d..ee6ea4d 100644 --- a/internal/client/mediaedge/source.go +++ b/internal/client/mediaedge/source.go @@ -2,126 +2,462 @@ package mediaedge import ( "errors" + "io" "sync" "sync/atomic" "time" + "github.com/TNTcraftHIM/Screener/internal/client/nativecapture" + "github.com/TNTcraftHIM/Screener/internal/media/encoded" + "github.com/TNTcraftHIM/Screener/internal/media/forwarding" + mediacodec "github.com/livekit/mediatransportutil/pkg/codec" + "github.com/pion/rtcp" "github.com/pion/rtp" "github.com/pion/webrtc/v4" ) -const maxAccessUnitBytes = 4 * 1024 * 1024 const videoClockRate = 90_000 const videoPacketMTU = 1200 const h264PayloadType = 102 const vp8PayloadType = 96 var ErrSourceCapacity = errors.New("native media source capacity is exhausted") -var ErrInvalidVideoTimestamp = errors.New("native video timestamp is not monotonic") + +// OutputPlan belongs to the capture input that produced it. The caller applies +// it to that exact stream after BeginFrame releases the source locks. +type OutputPlan struct { + Active []bool + RecoveryLayers []int + Bitrates []uint32 +} type Source struct { - engine *Engine - track *webrtc.TrackLocalStaticRTP - codec string - packetizer rtp.Packetizer - capacity int - requestKeyFrame func() + engine *Engine + codec string + media *forwarding.EncodedSource + formats []atomic.Uint64 + outputBitrates []uint32 + capacity int + requestKeyFrame func() + relayProfile *nativecapture.VideoProfile + relay *relayDerivation + groups map[int]*outputGroup + memberships map[groupConsumer]*outputGroup + groupRecovery atomic.Uint32 + recoveryRequests chan struct{} + recoveryStopped chan struct{} + inputPTS time.Duration + inputAt time.Time + hasInput bool - mu sync.Mutex - writeMu sync.Mutex - edges map[*Edge]bool - reservations int - localReservation bool - closed bool - hasTimestamp bool - lastTimestamp time.Duration - timestampRemainder float64 - rebasePending bool - rebaseInput time.Duration - rebaseOutput time.Duration - frames atomic.Uint64 - bytes atomic.Uint64 - format atomic.Uint64 + mu sync.Mutex + writeMu sync.Mutex + edges map[*Edge]bool + publications map[*Publication]struct{} + reservations int + localReservation bool + closed bool } func (source *Source) Codec() string { return source.codec } -func (source *Source) WriteVideo( - accessUnit []byte, - timestamp time.Duration, - duration time.Duration, -) error { - if len(accessUnit) == 0 || len(accessUnit) > maxAccessUnitBytes || - timestamp < 0 || duration <= 0 { - return errors.New("native video access unit is invalid") +func (source *Source) SetLowestLayerRateControlled(controlled bool) { + source.writeMu.Lock() + defer source.writeMu.Unlock() + source.media.SetLowestLayerRateControlled(controlled) + for slot, group := range source.groups { + if slot != 0 { + group.media.SetLowestLayerRateControlled(controlled) + } + } +} + +// The room supplies a relay ceiling, never a replacement capture authority. +// Recording it does not create an encoder or invoke the capture process. +func (source *Source) SetRelayProfile(profile nativecapture.VideoProfile) error { + if !profile.Valid() { + return errors.New("native relay profile is invalid") + } + source.mu.Lock() + if source.closed { + source.mu.Unlock() + return io.ErrClosedPipe + } + if source.relayProfile != nil && *source.relayProfile == profile { + source.mu.Unlock() + return nil + } + source.relayProfile = &profile + source.mu.Unlock() + if source.relay != nil { + source.relay.cancel() + source.writeMu.Lock() + defer source.writeMu.Unlock() + return source.configureRelayFormats(true) + } + return nil +} + +// BeginFrame runs once before outputs for an actual source input. Unconnected +// edges retain their capacity reservation but do not demand encoded output yet. +func (source *Source) BeginFrame(timestamp time.Duration) (OutputPlan, error) { + return source.beginFrame(timestamp, time.Now()) +} + +func (source *Source) beginFrame(timestamp time.Duration, at time.Time) (OutputPlan, error) { + source.writeMu.Lock() + var unavailable []*Edge + defer func() { + source.writeMu.Unlock() + closeEdges(unavailable) + }() + source.mu.Lock() + if source.closed { + source.mu.Unlock() + return OutputPlan{}, io.ErrClosedPipe + } + edges := make([]*Edge, 0, len(source.edges)) + for edge := range source.edges { + edges = append(edges, edge) + } + publications := make([]*Publication, 0, len(source.publications)) + for publication := range source.publications { + publications = append(publications, publication) + } + source.mu.Unlock() + if err := source.beginGroupFrame(timestamp, at); err != nil { + return OutputPlan{}, err + } + for _, edge := range edges { + if edge.State() != webrtc.PeerConnectionStateConnected { + continue + } + if err := edge.transport.SetConnected(); err != nil { + return OutputPlan{}, err + } + if source.unavailableDemand(edge) { + unavailable = append(unavailable, edge) + continue + } + edge.transport.Output.BeginFrame() + } + if len(source.formats) == 2 && len(source.outputBitrates) >= 2 { + plan, err := source.planGroups(source.collectDemands(edges, publications)) + for mask, slot := source.groupRecovery.Swap(0), 0; mask != 0; mask, slot = mask>>1, slot+1 { + if mask&1 != 0 && slot < len(plan.Active) && plan.Active[slot] { + plan.RecoveryLayers = append(plan.RecoveryLayers, slot) + } + } + return plan, err + } + plan := OutputPlan{Active: make([]bool, len(source.outputBitrates)), Bitrates: append([]uint32(nil), source.outputBitrates...)} + count := 0 + for _, edge := range edges { + if edge.State() == webrtc.PeerConnectionStateConnected { + count = max(count, edge.transport.RequiredActiveCount()) + } + } + for _, publication := range publications { + count = max(count, publication.RequiredActiveCount()) + } + for slot := 0; slot < min(count, len(plan.Active)); slot++ { + plan.Active[slot] = true + } + return plan, nil +} + +// ConfigureOutputs installs one validated capture generation's bitrate bounds. +// A zero slot subsequently means that producer has failed, not a new preset. +func (source *Source) ConfigureOutputs(ceilings []uint32) error { + if len(ceilings) < len(source.formats) || len(ceilings) > len(source.formats)+source.capacity || + len(source.formats) != 2 && len(ceilings) != len(source.formats) { + return errors.New("native output count does not match the source") + } + for _, bitrate := range ceilings { + if bitrate < 1000 { + return errors.New("native output bitrate is invalid") + } } source.writeMu.Lock() defer source.writeMu.Unlock() + source.mu.Lock() + defer source.mu.Unlock() + if source.closed { + return io.ErrClosedPipe + } + source.outputBitrates = append([]uint32(nil), ceilings...) + formats := make([]forwarding.LayerFormat, len(source.formats)) + for layer := range formats { + format := source.formats[layer].Load() + formats[layer] = forwarding.LayerFormat{Width: uint32(format >> 32), Height: uint32(format), Bitrate: ceilings[layer]} + } + if err := source.media.UpdateFormats(formats); err != nil { + return err + } + for slot, group := range source.groups { + group.budget = ceilings[slot] + if slot != 0 { + formats[0].Bitrate = ceilings[slot] + if err := group.media.UpdateFormats(formats); err != nil { + return err + } + } + } + return nil +} + +func (source *Source) DisableLayer(layer int) error { + retire, err := source.markLayerUnavailable(layer, nil) + if retire != nil { + retire() + } + return err +} + +func (source *Source) markLayerUnavailable(layer int, run *relayRun) (func(), error) { + if run != nil { + run.owner.mu.Lock() + defer run.owner.mu.Unlock() + } + source.writeMu.Lock() + defer source.writeMu.Unlock() + source.mu.Lock() + if run != nil && (run.owner.run != run || source.relayProfile != run.plan.profile || + source.formats[len(source.formats)-1].Load() != run.plan.format) { + source.mu.Unlock() + return nil, nil + } + if source.closed || layer < 0 || layer >= len(source.outputBitrates) { + source.mu.Unlock() + return nil, errors.New("native output layer is unavailable") + } + source.outputBitrates[layer] = 0 + if layer == 0 { + source.media.SetLowestLayerRateControlled(false) + } + publications := make([]*Publication, 0, len(source.publications)) + group := source.groups[layer] + for publication := range source.publications { + if layer == 1 || group == nil && layer < len(source.formats) || group != nil && + (publication.transport.CurrentSource() == group.media.Source || source.memberships[publication.transport] == group) { + publications = append(publications, publication) + } + } + var affected []*Edge + for edge := range source.edges { + state := edge.transport.Output.State() + _, observed := edge.transport.TargetBitrate() + selectedLayer := int32(layer) + if group != nil { + if source.memberships[edge.transport] == group { + affected = append(affected, edge) + continue + } + if edge.transport.CurrentSource() != group.media.Source { + continue + } + selectedLayer = 0 + } + if state.Target == selectedLayer || state.Current == selectedLayer || + (observed && state.Paused && state.Prepare == selectedLayer) || + (state.Target < 0 && state.Current < 0 && edge.transport.Output.MaxLayer().Spatial == selectedLayer) { + affected = append(affected, edge) + } + } + source.mu.Unlock() + media, logical := source.media, layer + if group != nil { + media, logical = group.media, 0 + group.active = false + media.SetLowestLayerRateControlled(false) + } + if logical < len(source.formats) { + if tracker := media.StreamTrackerManager().GetTracker(int32(logical)); tracker != nil { + tracker.SetPaused(true) + } + } + // An unavailable encoder retires its consumers through the existing edge + // lifecycle. Healthy sibling outputs are neither stopped nor restarted. + return func() { + closeEdges(affected) + // Publication metadata declares every slot, including paused or preparing + // uploads. None may publish a descriptor containing a failed encoding. + for _, publication := range publications { + _ = publication.Close() + } + }, nil +} + +func (source *Source) WriteVideo(layer int, frame encoded.Frame) error { + source.writeMu.Lock() + defer source.writeMu.Unlock() + return source.writeVideo(layer, frame) +} + +// Caller serializes capture/source generation with this output. +func (source *Source) writeVideo(layer int, frame encoded.Frame) error { source.mu.Lock() closed := source.closed source.mu.Unlock() if closed { - return errors.New("native media source is closed") + return io.ErrClosedPipe } - if source.rebasePending { - source.rebasePending = false - source.rebaseInput = timestamp - source.rebaseOutput = source.lastTimestamp + duration - timestamp = source.rebaseOutput - } else if source.rebaseOutput > 0 { - if timestamp < source.rebaseInput { - return ErrInvalidVideoTimestamp + if len(source.outputBitrates) > 0 && + (layer < 0 || layer >= len(source.outputBitrates) || source.outputBitrates[layer] == 0) { + return errors.New("native output layer is unavailable") + } + if len(frame.Data) == 0 || frame.Duration <= 0 || !source.hasInput || frame.PTS < 0 || frame.PTS > source.inputPTS { + return errors.New("encoded output has no matching source input") + } + if len(source.formats) != 2 { + return source.media.WriteFrame(layer, frame) + } + if layer == 1 { + if err := source.media.WriteFrame(1, frame); err != nil { + return err } - timestamp = source.rebaseOutput + timestamp - source.rebaseInput - } - if source.hasTimestamp { - if timestamp <= source.lastTimestamp { - return ErrInvalidVideoTimestamp + for slot, group := range source.groups { + if slot != 0 && group.active && frame.PTS >= group.start { + if err := group.media.WriteFrame(1, frame); err != nil { + return err + } + } } - ticksFloat := (timestamp-source.lastTimestamp).Seconds()*videoClockRate + - source.timestampRemainder - ticks := uint64(ticksFloat) - source.timestampRemainder = ticksFloat - float64(ticks) - source.packetizer.SkipSamples(uint32(ticks)) - } else { - source.hasTimestamp = true + if frame.Recovery { + source.installGroup(nil, true) + } + return nil } - source.lastTimestamp = timestamp - packets := source.packetizer.Packetize(accessUnit, 0) - if len(packets) == 0 { - return errors.New("native video access unit produced no RTP packets") + group := source.groups[layer] + if group == nil { + if layer == 0 { + return source.media.WriteFrame(0, frame) + } + return nil } - source.frames.Add(1) - source.bytes.Add(uint64(len(accessUnit))) - var result error - for _, packet := range packets { - result = errors.Join(result, source.track.WriteRTP(packet)) + if !group.active || frame.PTS < group.start { + return nil } - return result + if err := group.media.WriteFrame(0, frame); err != nil { + return err + } + if frame.Recovery { + source.installGroup(group, false) + } + return nil } -// WriteRTP forwards one already encoded video packet without decoding or -// re-encoding it. TrackLocalStaticRTP rewrites the negotiated SSRC and payload -// type independently for every bound edge. +// WriteRTP preserves the received representation. Optional lower derivation is +// fed without waiting for its decoder, encoder, or process pipe. func (source *Source) WriteRTP(packet *rtp.Packet) error { if packet == nil { return errors.New("native video RTP packet is invalid") } source.writeMu.Lock() - defer source.writeMu.Unlock() source.mu.Lock() - closed := source.closed + if source.closed { + source.mu.Unlock() + source.writeMu.Unlock() + return io.ErrClosedPipe + } source.mu.Unlock() - if closed { - return errors.New("native media source is closed") + layer := len(source.formats) - 1 + recovery := false + if source.relay != nil { + var size mediacodec.VideoSize + if source.codec == "h264" { + size = mediacodec.ExtractH264VideoSize(packet.Payload) + } else { + var header mediacodec.VP8 + if header.Unmarshal(packet.Payload) == nil { + size = mediacodec.ExtractVP8VideoSize(&header, packet.Payload) + } + } + if size.Width > 0 && size.Height > 0 { + recovery = true + format := uint64(size.Width)<<32 | uint64(size.Height) + if source.formats[layer].Swap(format) != format { + _ = source.configureRelayFormats(false) + } + } } - // Forward RTP padding to preserve sequence continuity without counting it as video. - if packet.Marker && len(packet.Payload) > 0 { - source.frames.Add(1) - } - source.bytes.Add(uint64(len(packet.Payload))) forwarded := connectionNeutralRTP(packet) - return source.track.WriteRTP(&forwarded) + forwarded.SSRC = uint32(layer + 1) + forwarded.PayloadType = uint8(videoCodecs[source.codec].PayloadType) + err := source.media.Source.WriteRTP(layer, &forwarded) + for slot, group := range source.groups { + if slot != 0 && group.active { + if groupErr := group.media.Source.WriteRTP(layer, &forwarded); err == nil { + err = groupErr + } + } + } + if recovery && err == nil { + source.installGroup(nil, true) + } + var unavailable []*Edge + if packet.Marker { + source.mu.Lock() + for edge := range source.edges { + if edge.State() == webrtc.PeerConnectionStateConnected && source.unavailableDemand(edge) { + unavailable = append(unavailable, edge) + } + } + source.mu.Unlock() + } + source.writeMu.Unlock() + closeEdges(unavailable) + if source.relay != nil { + source.relay.push(packet) + } + return err +} + +// Called with writeMu held. Preparation is advisory; the source owns whether +// that requested codec output still exists after a producer failure. +func (source *Source) unavailableDemand(edge *Edge) bool { + state := edge.transport.Output.State() + selected := state.Target + if !state.Paused { + selected = max(selected, state.Current) + } + physical := selected + if selected == 0 { + if group := source.groupForMedia(edge.transport.CurrentSource()); group != nil { + physical = int32(group.slot) + } + } + if selected >= 0 && !source.failedOutput(physical) { + return false + } + fallback := int32(edge.transport.RequiredActiveCount() - 1) + if fallback == 0 { + if group := source.groupForMedia(edge.transport.CurrentSource()); group != nil { + fallback = int32(group.slot) + } + } + return source.failedOutput(fallback) +} + +func (source *Source) failedOutput(layer int32) bool { + return layer >= 0 && int(layer) < len(source.outputBitrates) && source.outputBitrates[layer] == 0 +} + +func (source *Source) SenderReport(report *rtcp.SenderReport) error { + source.writeMu.Lock() + defer source.writeMu.Unlock() + correlated := *report + layer := len(source.formats) - 1 + correlated.SSRC = uint32(layer + 1) + err := source.media.SenderReport(layer, &correlated) + for slot, group := range source.groups { + if slot != 0 && group.active { + if groupErr := group.media.SenderReport(layer, &correlated); err == nil { + err = groupErr + } + } + } + return err } func connectionNeutralRTP(packet *rtp.Packet) rtp.Packet { @@ -137,33 +473,55 @@ func connectionNeutralRTP(packet *rtp.Packet) rtp.Packet { func (source *Source) BeginGeneration() { source.writeMu.Lock() - source.rebasePending = true - source.rebaseInput = 0 - source.rebaseOutput = 0 - source.writeMu.Unlock() -} - -func (source *Source) SetFormat(width, height uint32) { - source.format.Store(uint64(width)<<32 | uint64(height)) -} - -type sourceSnapshot struct { - frames uint64 - bytes uint64 - width uint32 - height uint32 -} - -func (source *Source) snapshot() sourceSnapshot { - format := source.format.Load() - return sourceSnapshot{ - frames: source.frames.Load(), - bytes: source.bytes.Load(), - width: uint32(format >> 32), - height: uint32(format), + defer source.writeMu.Unlock() + source.media.BeginGeneration() + source.hasInput = false + for slot, group := range source.groups { + group.start = 0 + if slot != 0 { + group.media.BeginGeneration() + } } } +func (source *Source) SetFormat(layer int, width, height uint32) error { + source.writeMu.Lock() + defer source.writeMu.Unlock() + return source.setFormat(layer, width, height) +} + +func (source *Source) setFormat(layer int, width, height uint32) error { + if layer < 0 || layer >= max(len(source.formats), len(source.outputBitrates)) || width == 0 || height == 0 { + return errors.New("native output format is invalid") + } + source.mu.Lock() + defer source.mu.Unlock() + if source.closed { + return io.ErrClosedPipe + } + format := uint64(width)<<32 | uint64(height) + if layer >= len(source.formats) { + if group := source.groups[layer]; group != nil { + return group.media.UpdateDimensions(0, width, height) + } + return nil + } + if err := source.media.UpdateDimensions(layer, width, height); err != nil { + return err + } + source.formats[layer].Store(format) + if layer == 1 { + for slot, group := range source.groups { + if slot != 0 { + if err := group.media.UpdateDimensions(1, width, height); err != nil { + return err + } + } + } + } + return nil +} + func (source *Source) reserve(local bool) error { source.mu.Lock() defer source.mu.Unlock() @@ -196,6 +554,8 @@ func (source *Source) reserve(local bool) error { } func (source *Source) attach(edge *Edge, local bool) error { + source.writeMu.Lock() + defer source.writeMu.Unlock() source.mu.Lock() defer source.mu.Unlock() if local { @@ -206,6 +566,16 @@ func (source *Source) attach(edge *Edge, local bool) error { if source.closed { return errors.New("native media source is closed") } + if len(source.outputBitrates) > 0 { + highest := len(source.formats) - 1 + for highest >= 0 && source.outputBitrates[highest] == 0 { + highest-- + } + if highest < 0 { + return errors.New("native media source has no available output") + } + edge.transport.Output.SetMaxSpatialLayer(int32(highest)) + } source.edges[edge] = local return nil } @@ -221,14 +591,55 @@ func (source *Source) releaseReservation(local bool) { } func (source *Source) detach(edge *Edge) { + source.writeMu.Lock() source.mu.Lock() delete(source.edges, edge) source.mu.Unlock() + source.writeMu.Unlock() + if source.relay != nil && !source.relayPlan().wanted { + source.relay.cancel() + } + source.RequestRecoveryFrame() +} + +func (source *Source) requestLayerKeyFrame(layer int) { + if source.relay != nil && layer < len(source.formats)-1 { + source.relay.recovery.Store(true) + } + source.groupRecovery.Or(uint32(1) << layer) + source.RequestRecoveryFrame() } func (source *Source) RequestRecoveryFrame() { if source.requestKeyFrame != nil { - source.requestKeyFrame() + select { + case source.recoveryRequests <- struct{}{}: + default: + } + } +} + +// One coalescing notification owner per source, never per encoder or frame. +// Producer I/O cannot reenter a source lock from a forwarding callback. +func (source *Source) runRecovery() { + var engineDone <-chan struct{} + if source.engine.ctx != nil { + engineDone = source.engine.ctx.Done() + } + for { + select { + case <-engineDone: + return + case <-source.recoveryStopped: + return + case <-source.recoveryRequests: + source.mu.Lock() + closed := source.closed + source.mu.Unlock() + if !closed { + source.requestKeyFrame() + } + } } } @@ -239,13 +650,33 @@ func (source *Source) Close() error { return nil } source.closed = true + if source.recoveryStopped != nil { + close(source.recoveryStopped) + } + publications := make([]*Publication, 0, len(source.publications)) + for publication := range source.publications { + publications = append(publications, publication) + } edges := make([]*Edge, 0, len(source.edges)) for edge := range source.edges { edges = append(edges, edge) } source.mu.Unlock() - for _, edge := range edges { - _ = edge.Close() + // Close transports before waiting on their possible writes into this group. + closeEdges(edges) + for _, publication := range publications { + _ = publication.Close() } + if source.relay != nil { + source.relay.close() + } + source.writeMu.Lock() + for slot, group := range source.groups { + if slot != 0 { + group.media.Close() + } + } + source.media.Close() + source.writeMu.Unlock() return nil } diff --git a/internal/client/mediaedge/source_test.go b/internal/client/mediaedge/source_test.go new file mode 100644 index 0000000..33358fa --- /dev/null +++ b/internal/client/mediaedge/source_test.go @@ -0,0 +1,112 @@ +package mediaedge + +import ( + "errors" + "io" + "testing" + "time" + + "github.com/TNTcraftHIM/Screener/internal/media/encoded" + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/streamtracker" + "google.golang.org/protobuf/proto" +) + +func TestSourceFormatAdaptationPreservesLiveLayers(t *testing.T) { + // A source needs no socket or codec process to exercise its RTP state. + source, err := (&Engine{}).NewSource("vp8", 2, 2, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = source.Close() }) + check := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + check(source.SetFormat(0, 640, 360)) + check(source.SetFormat(1, 1280, 720)) + check(source.ConfigureOutputs([]uint32{300_000, 1_200_000})) + check(source.media.UpdateDimensions(0, 320, 180)) + check(source.SetFormat(0, 640, 360)) + if source.media.TrackInfo().Layers[0].Width != 640 { + t.Fatal("unchanged actual dimensions did not replace stale metadata") + } + initialInfo := source.media.TrackInfo() + buffers := source.media.GetAllBuffers() + trackers := source.media.StreamTrackerManager() + const period = time.Second / 25 + var pts time.Duration + write := func() { + t.Helper() + // This check owns input explicitly; no consumer is present to request + // codec activation through the separate demand planner. + check(source.beginGroupFrame(pts, time.Now())) + for layer := range 2 { + check(source.WriteVideo(layer, encoded.Frame{Data: sfu.VP8KeyFrame8x8, PTS: pts, Duration: period})) + } + pts += period + } + ticker := time.NewTicker(period) + defer ticker.Stop() + deadline := time.NewTimer(6 * time.Second) + defer deadline.Stop() + for { + select { + case <-ticker.C: + case <-deadline.C: + t.Fatal("live layers produced no measured bitrate") + } + write() + _, rates := source.media.GetLayeredBitrate() + if rates[0][0] > 0 && rates[1][0] > 0 { + break + } + } + want := source.media.TrackInfo() + for _, width := range []uint32{320, 320, 640} { + before := source.media.TrackInfo() + clock := buffers[0].GetSenderReportData() + check(source.SetFormat(0, width, width*9/16)) + want.Layers[0].Width, want.Layers[0].Height = width, width*9/16 + want.Codecs[0].Layers[0].Width, want.Codecs[0].Layers[0].Height = width, width*9/16 + if !proto.Equal(source.media.TrackInfo(), want) || source.formats[0].Load() != uint64(width)<<32|uint64(width*9/16) { + t.Fatal("adapted dimensions did not reach layer and codec metadata") + } + before.Layers[1].Width = 1 + if source.media.TrackInfo().Layers[1].Width != 1280 || initialInfo.Layers[0].Width != 640 { + t.Fatal("dimension metadata escaped its immutable snapshot") + } + for layer := range 2 { + tracker := trackers.GetTracker(int32(layer)) + if tracker == nil || tracker.Status() != streamtracker.StreamStatusActive || tracker.BitrateTemporalCumulative()[0] == 0 || + source.media.GetAllBuffers()[layer] != buffers[layer] { + t.Fatalf("dimension update reset live layer %d", layer) + } + } + if !proto.Equal(clock, buffers[0].GetSenderReportData()) { + t.Fatal("dimension update changed the source clock or counters") + } + write() + after := buffers[0].GetSenderReportData() + if after.Packets != clock.Packets+1 || after.Octets <= clock.Octets || after.RtpTimestamp-clock.RtpTimestamp != 3600 || + after.NtpTimestamp <= clock.NtpTimestamp { + t.Fatal("adapted output did not retain the source RTP timeline") + } + } + for _, format := range [][3]int{{-1, 320, 180}, {2, 320, 180}, {0, 0, 180}, {0, 320, 0}} { + if source.SetFormat(format[0], uint32(format[1]), uint32(format[2])) == nil || + source.media.UpdateDimensions(format[0], uint32(format[1]), uint32(format[2])) == nil { + t.Fatal("invalid output dimensions were accepted") + } + } + if !proto.Equal(source.media.TrackInfo(), want) { + t.Fatal("invalid format changed source metadata") + } + check(source.Close()) + if !errors.Is(source.SetFormat(0, 640, 360), io.ErrClosedPipe) || + !errors.Is(source.media.UpdateDimensions(0, 640, 360), sfu.ErrReceiverClosed) { + t.Fatal("closed source accepted new dimensions") + } +} diff --git a/internal/client/mediaedge/testdata/gcc_feedback_probe.go b/internal/client/mediaedge/testdata/gcc_feedback_probe.go new file mode 100644 index 0000000..f7e4d74 --- /dev/null +++ b/internal/client/mediaedge/testdata/gcc_feedback_probe.go @@ -0,0 +1,204 @@ +package main + +import ( + "encoding/binary" + "encoding/json" + "flag" + "fmt" + "io" + "os" + "time" + + "github.com/pion/interceptor" + "github.com/pion/interceptor/pkg/gcc" + "github.com/pion/interceptor/pkg/twcc" + "github.com/pion/logging" + "github.com/pion/rtp" +) + +const payloadBytes = 1200 + +type arrival struct { + sequence uint16 + at time.Time +} + +type arm struct { + name string + bwe *gcc.SendSideBWE + writer interceptor.RTPWriter + recorder *twcc.Recorder + capacity int + lastArrival time.Time + queue []arrival + sourceBudget float64 + sequence uint16 + sent int + received int + dropped int + maxQueue time.Duration +} + +type observation struct { + Second float64 `json:"second"` + Arm string `json:"arm"` + Phase string `json:"phase"` + CapacityBps int `json:"capacityBps"` + TargetBps int `json:"targetBps"` + SourceBps int `json:"sourceBps"` + SentBps int `json:"sentBps"` + ReceivedBps int `json:"receivedBps"` + Dropped int `json:"droppedPackets"` + MaxQueueMs float64 `json:"maxQueueMs"` + Estimator map[string]any `json:"estimator"` +} + +func newArm(name string) (*arm, error) { + logger := logging.NewDefaultLoggerFactory() + logger.Writer = io.Discard + bwe, err := gcc.NewSendSideBWE( + gcc.SendSideBWEInitialBitrate(5_000_000), + gcc.SendSideBWEPacer(gcc.NewNoOpPacer()), + gcc.WithLoggerFactory(logger), + ) + if err != nil { + return nil, err + } + a := &arm{name: name, bwe: bwe, recorder: twcc.NewRecorder(2)} + info := &interceptor.StreamInfo{SSRC: 1, RTPHeaderExtensions: []interceptor.RTPHeaderExtension{ + {URI: "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01", ID: 1}, + }} + sink := interceptor.RTPWriterFunc(func(header *rtp.Header, payload []byte, _ interceptor.Attributes) (int, error) { + now := time.Now() + delivery := now.Add(20 * time.Millisecond) + if a.lastArrival.After(delivery) { + delivery = a.lastArrival + } + delivery = delivery.Add(time.Duration(float64(len(payload)*8) / float64(a.capacity) * float64(time.Second))) + a.sent += len(payload) + if delivery.Sub(now) > 120*time.Millisecond { + a.dropped++ + return len(payload), nil + } + a.lastArrival = delivery + a.maxQueue = max(a.maxQueue, delivery.Sub(now)) + a.queue = append(a.queue, arrival{sequence: binary.BigEndian.Uint16(header.GetExtension(1)), at: delivery}) + return len(payload), nil + }) + twccHeader := &twcc.HeaderExtensionInterceptor{} + a.writer = twccHeader.BindLocalStream(info, bwe.AddStream(info, sink)) + return a, nil +} + +func main() { + if err := run(); err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } +} + +func run() error { + frameBursts := flag.Bool("frame-bursts", false, "emit the payload budget in30fps frame bursts") + flag.Parse() + var arms []*arm + for _, name := range []string{"low-layer-capped", "follows-budget"} { + a, err := newArm(name) + if err != nil { + return err + } + arms = append(arms, a) + defer a.bwe.Close() + } + start := time.Now() + lastSource, nextSource := start, start + lastReport, lastFeedback := start, start + ticker := time.NewTicker(5 * time.Millisecond) + defer ticker.Stop() + var rows []observation + payload := make([]byte, payloadBytes) + for range ticker.C { + now := time.Now() + elapsed := now.Sub(start) + if elapsed >= 40*time.Second { + break + } + phase, capacity := "high", 6_000_000 + if elapsed >= 5*time.Second && elapsed < 10*time.Second { + phase, capacity = "constrained", 500_000 + } else if elapsed >= 10*time.Second { + phase = "released" + } + emitSource := !*frameBursts || !now.Before(nextSource) + for _, a := range arms { + a.capacity = capacity + for len(a.queue) > 0 && !a.queue[0].at.After(now) { + packet := a.queue[0] + a.queue = a.queue[1:] + a.recorder.Record(1, packet.sequence, packet.at.Sub(start).Microseconds()) + a.received += payloadBytes + } + sourceBps := min(5_000_000, a.bwe.GetTargetBitrate()) + if phase == "released" && a.name == "low-layer-capped" { + sourceBps = min(sourceBps, 1_250_000) + } + if emitSource { + a.sourceBudget += now.Sub(lastSource).Seconds() * float64(sourceBps) / 8 + } + for a.sourceBudget >= payloadBytes { + a.sourceBudget -= payloadBytes + header := &rtp.Header{Version: 2, SSRC: 1, PayloadType: 96, + SequenceNumber: a.sequence, Timestamp: uint32(elapsed.Seconds() * 90_000)} + a.sequence++ + if _, err := a.writer.Write(header, payload, nil); err != nil { + return err + } + } + if now.Sub(lastFeedback) >= 100*time.Millisecond { + if err := a.bwe.WriteRTCP(a.recorder.BuildFeedbackPacket(), nil); err != nil { + return err + } + } + if now.Sub(lastReport) >= time.Second { + window := now.Sub(lastReport).Seconds() + row := observation{ + Second: elapsed.Seconds(), Arm: a.name, Phase: phase, CapacityBps: capacity, + TargetBps: a.bwe.GetTargetBitrate(), SourceBps: sourceBps, + SentBps: int(float64(a.sent*8) / window), ReceivedBps: int(float64(a.received*8) / window), + Dropped: a.dropped, MaxQueueMs: float64(a.maxQueue.Microseconds()) / 1000, + Estimator: a.bwe.GetStats(), + } + rows = append(rows, row) + fmt.Printf("%5.1fs %-16s %-11s target=%7d source=%7d received=%7d loss=%d state=%v\n", + row.Second, row.Arm, row.Phase, row.TargetBps, row.SourceBps, row.ReceivedBps, row.Dropped, row.Estimator["state"]) + a.sent, a.received, a.dropped, a.maxQueue = 0, 0, 0, 0 + } + } + if now.Sub(lastFeedback) >= 100*time.Millisecond { + lastFeedback = now + } + if now.Sub(lastReport) >= time.Second { + lastReport = now + } + if emitSource { + lastSource = now + framePeriod := time.Second / 30 + nextSource = start.Add((elapsed/framePeriod + 1) * framePeriod) + } + } + name, cadence := "build/embedded-media/gcc-feedback.json", "5ms packet budget" + if *frameBursts { + name, cadence = "build/embedded-media/gcc-feedback-frames.json", "30fps frame bursts" + } + output, err := os.Create(name) + if err != nil { + return err + } + defer output.Close() + return json.NewEncoder(output).Encode(map[string]any{ + "pionInterceptor": "v0.1.47", "kind": "in-memory payload-rate constrained GCC/TWCC feedback", + "initialBitrate": 5_000_000, "lowOutputCap": 1_250_000, + "linkPropagationMs": 20, "linkMaxDelayMs": 120, "feedbackMs": 100, + "pacer": "NoOpPacer", "observations": rows, + "sourceCadence": cadence, + }) +} diff --git a/internal/client/nativeaudio/encoder.go b/internal/client/nativeaudio/encoder.go index 9e5a94d..d037d08 100644 --- a/internal/client/nativeaudio/encoder.go +++ b/internal/client/nativeaudio/encoder.go @@ -26,6 +26,8 @@ type Encoder struct { packet [MaxPacketBytes]byte } +func (encoder *Encoder) Bitrate() int { return encoder.codec.Bitrate() } + func (encoder *Encoder) SetBitrate(bitrate int) error { if encoder == nil || encoder.codec == nil || bitrate < 6_000 || bitrate > 510_000 { return errors.New("native audio bitrate is outside the Opus range") diff --git a/internal/client/nativecapture/adaptive_fixture_test.go b/internal/client/nativecapture/adaptive_fixture_test.go new file mode 100644 index 0000000..fc554f3 --- /dev/null +++ b/internal/client/nativecapture/adaptive_fixture_test.go @@ -0,0 +1,159 @@ +package nativecapture + +import ( + "context" + "encoding/hex" + "encoding/json" + "errors" + "io" + "os" + "testing" + "time" +) + +// Opt-in real capture-process check; normal tests never start a codec or GPU. +func TestAdaptiveOutputFixture(t *testing.T) { + executable, fixture := os.Getenv("SCREENER_NATIVE_CAPTURE"), os.Getenv("SCREENER_ADAPTIVE_INPUT") + if executable == "" || fixture == "" { + t.Skip("set SCREENER_NATIVE_CAPTURE and SCREENER_ADAPTIVE_INPUT") + } + codec := os.Getenv("SCREENER_ADAPTIVE_CODEC") + if codec == "" { + codec = "vp8" + } + kind := FrameVP8 + if codec == "h264" { + kind = FrameH264 + } + input, err := os.Open(fixture) + if err != nil { + t.Fatal(err) + } + defer input.Close() + var frames []Frame + decoder := json.NewDecoder(input) + for { + var row struct { + Width, Height uint32 + Recovery bool + DataHex string + } + if err = decoder.Decode(&row); err == io.EOF { + break + } + if err != nil || len(frames) >= 180 || len(row.DataHex) > maxMediaBytes*2 { + t.Fatalf("invalid bounded input: %v", err) + } + data, err := hex.DecodeString(row.DataHex) + if err != nil || row.Width != 640 || row.Height != 360 { + t.Fatal("expected 640x360 encoded input") + } + frames = append(frames, Frame{Kind: kind, KeyFrame: row.Recovery, + Width: row.Width, Height: row.Height, Data: data}) + } + if len(frames) == 0 || !frames[0].KeyFrame { + t.Fatal("input must begin with recovery") + } + ctx, cancel := context.WithTimeout(t.Context(), 85*time.Second) + defer cancel() + stream, err := StartEncodedVideo(ctx, executable, EncodedVideoOptions{ + Codec: codec, Preference: "maintain-framerate", + Outputs: []OutputProfile{{Width: 640, Height: 360, Framerate: 30, Bitrate: 2_000_000}}, + }) + if err != nil { + t.Fatal(err) + } + defer stream.Close() + if err = stream.SetOutputActive(0, true); err != nil { + t.Fatal(err) + } + readDone := make(chan error, 1) + timelineError := errors.New("adaptive output violated its input timeline") + var output []Frame + go func() { + var begin, previous time.Duration + for { + frame, err := stream.Read() + if err != nil { + readDone <- err + return + } + if frame.Kind == FrameBegin && begin == 0 { + begin = frame.Timestamp + } + if frame.Kind == FrameLayerUnavailable { + readDone <- errors.New("capture layer unavailable: " + string(frame.Data)) + return + } + if frame.Kind != kind { + continue + } + if frame.Timestamp < begin || frame.Timestamp <= previous || len(output) >= 2400 { + readDone <- timelineError + return + } + previous = frame.Timestamp + output = append(output, frame) + } + }() + ticker := time.NewTicker(time.Second / 30) + defer ticker.Stop() + for index := 0; index < 76*30; index++ { + select { + case err := <-readDone: + t.Fatalf("capture stopped during adaptation: %v", err) + case <-ctx.Done(): + t.Fatal(ctx.Err()) + case <-ticker.C: + } + if index == 6*30 || index == 21*30 { + budget := uint32(100_000) + if index == 21*30 { + budget = 2_000_000 + } + if err = stream.SetOutputBitrate(0, budget); err != nil { + t.Fatal(err) + } + } + frame := frames[index%len(frames)] + // Preserve a non-microsecond-aligned input anchor through WebRTC. + frame.Timestamp = time.Duration(index+1)*time.Second/30 + 100*time.Nanosecond + frame.Duration = time.Second / 30 + if err = stream.WriteFrame(frame); err != nil { + t.Fatal(err) + } + } + if err = stream.Close(); err != nil { + t.Fatal(err) + } + if err = <-readDone; err == timelineError { + t.Fatal(err) + } + var healthy, reduced, restored int + type interval struct { + Frames, Bytes int + Width, Height uint32 + } + intervals := make([]interval, 16) + for _, frame := range output { + window := &intervals[min(int(frame.Timestamp/(5*time.Second)), len(intervals)-1)] + window.Frames++ + window.Bytes += len(frame.Data) + window.Width, window.Height = frame.Width, frame.Height + switch { + case frame.Timestamp < 6*time.Second && frame.Width == 640: + healthy++ + case frame.Timestamp >= 6*time.Second && frame.Timestamp < 21*time.Second && frame.Width < 640: + reduced++ + case frame.Timestamp >= 21*time.Second && frame.Width == 640: + restored++ + } + } + for index, window := range intervals { + t.Logf("t=%ds frames=%d bytes=%d final=%dx%d", index*5, window.Frames, window.Bytes, window.Width, window.Height) + } + t.Logf("codec=%s healthy=%d reduced=%d restored=%d total=%d", codec, healthy, reduced, restored, len(output)) + if healthy == 0 || reduced == 0 || restored == 0 { + t.Fatal("real output did not preserve delivery through adapt/recover") + } +} diff --git a/internal/client/nativecapture/probe.go b/internal/client/nativecapture/probe.go index 6748392..00408ad 100644 --- a/internal/client/nativecapture/probe.go +++ b/internal/client/nativecapture/probe.go @@ -16,7 +16,7 @@ import ( ) const ( - probeProtocol = 4 + probeProtocol = 7 probeTimeout = 3 * time.Second maxProbeOutputBytes = 64 * 1024 maxProbeErrorBytes = 4 * 1024 diff --git a/internal/client/nativecapture/probe_test.go b/internal/client/nativecapture/probe_test.go index 9e6f5e8..cf5aadd 100644 --- a/internal/client/nativecapture/probe_test.go +++ b/internal/client/nativecapture/probe_test.go @@ -5,7 +5,7 @@ import ( "testing" ) -const validProbe = `{"protocol":4,"platform":"windows","platformBuild":"26200","videoCapture":true,"processAudio":true,"systemAudio":true,"softwareVP8":true,"adapters":[{"index":0,"name":"GPU","identity":"0x0:0x1","hardwareH264":[{"index":0,"name":"H264","identity":"{encoder}"}]}]}` +const validProbe = `{"protocol":7,"platform":"windows","platformBuild":"26200","videoCapture":true,"processAudio":true,"systemAudio":true,"softwareVP8":true,"adapters":[{"index":0,"name":"GPU","identity":"0x0:0x1","hardwareH264":[{"index":0,"name":"H264","identity":"{encoder}"}]}]}` func TestDecodeProbeSeparatesCaptureAudioAndHardwareEncode(t *testing.T) { capabilities, err := decodeProbe([]byte(validProbe)) @@ -21,8 +21,8 @@ func TestDecodeProbeSeparatesCaptureAudioAndHardwareEncode(t *testing.T) { func TestDecodeProbeRejectsUnknownTrailingAndDuplicateData(t *testing.T) { tests := []string{ - strings.Replace(validProbe, `"protocol":4`, `"protocol":4,"extra":true`, 1), - strings.Replace(validProbe, `"protocol":4`, `"protocol":1`, 1), + strings.Replace(validProbe, `"protocol":7`, `"protocol":7,"extra":true`, 1), + strings.Replace(validProbe, `"protocol":7`, `"protocol":6`, 1), validProbe + `{}`, strings.Replace(validProbe, `"adapters":[`, `"adapters":[{"index":0,"name":"GPU 2","identity":"0x0:0x2","hardwareH264":[]},`, 1), } @@ -35,7 +35,7 @@ func TestDecodeProbeRejectsUnknownTrailingAndDuplicateData(t *testing.T) { func TestSummaryDoesNotConflateVideoAudioAndEncoder(t *testing.T) { capabilities, err := decodeProbe([]byte( - `{"protocol":4,"platform":"windows","platformBuild":"19045","videoCapture":true,"processAudio":false,"systemAudio":true,"adapters":[{"index":0,"name":"GPU","identity":"0x0:0x1","hardwareH264":[]}]}`, + `{"protocol":7,"platform":"windows","platformBuild":"19045","videoCapture":true,"processAudio":false,"systemAudio":true,"adapters":[{"index":0,"name":"GPU","identity":"0x0:0x1","hardwareH264":[]}]}`, )) if err != nil { t.Fatal(err) diff --git a/internal/client/nativecapture/protocol.go b/internal/client/nativecapture/protocol.go index a101baf..bf6decc 100644 --- a/internal/client/nativecapture/protocol.go +++ b/internal/client/nativecapture/protocol.go @@ -5,66 +5,150 @@ import ( "errors" "io" "time" + "unicode/utf8" + + "github.com/TNTcraftHIM/Screener/internal/media/encoded" ) const ( - envelopeHeaderBytes = 28 - maxMediaBytes = 1024 * 1024 + envelopeHeaderBytes = 32 + maxMediaBytes = encoded.MaxAccessUnitBytes maxStatusBytes = 4 * 1024 + maxControlBytes = 64 ) type FrameKind uint8 const ( - FramePCM FrameKind = 1 - FrameH264 FrameKind = 2 - FrameStatus FrameKind = 3 - FrameVP8 FrameKind = 4 + FramePCM FrameKind = 1 + FrameH264 FrameKind = 2 + FrameStatus FrameKind = 3 + FrameVP8 FrameKind = 4 + FrameBegin FrameKind = 5 + FrameLayerUnavailable FrameKind = 6 + FrameControl FrameKind = 7 ) type Frame struct { - Kind FrameKind + Kind FrameKind + // KeyFrame denotes independent recovery, including H264 SPS/PPS when needed. KeyFrame bool + Layer int + Width uint32 + Height uint32 Timestamp time.Duration Duration time.Duration Data []byte } func readFrame(input io.Reader) (Frame, error) { - header := make([]byte, envelopeHeaderBytes) - if _, err := io.ReadFull(input, header); err != nil { + var header [envelopeHeaderBytes]byte + if _, err := io.ReadFull(input, header[:]); err != nil { return Frame{}, err } - kind := FrameKind(header[5]) - size := binary.BigEndian.Uint32(header[24:]) - maximum := uint32(maxMediaBytes) - if kind == FrameStatus { - maximum = maxStatusBytes + frame, size, err := parseFrameHeader(header[:]) + if err != nil { + return Frame{}, err } - if string(header[:4]) != "SMED" || header[4] != 1 || header[7] != 0 || - (kind != FramePCM && kind != FrameH264 && kind != FrameVP8 && kind != FrameStatus) || - size == 0 || size > maximum { - return Frame{}, errors.New("native capture frame header is invalid") + frame.Data = make([]byte, size) + if _, err = io.ReadFull(input, frame.Data); err != nil { + return Frame{}, err + } + if err = validateFrameData(frame); err != nil { + return Frame{}, err + } + return frame, nil +} + +func parseFrameHeader(header []byte) (Frame, uint32, error) { + kind := FrameKind(header[5]) + size := binary.BigEndian.Uint32(header[28:32]) + maximum := uint32(maxMediaBytes) + if kind == FrameStatus || kind == FrameLayerUnavailable { + maximum = maxStatusBytes + } else if kind == FrameControl { + maximum = maxControlBytes + } + if string(header[:4]) != "SMED" || header[4] != 2 || + kind < FramePCM || kind > FrameControl || size > maximum || + (kind == FrameBegin) != (size == 0) { + return Frame{}, 0, errors.New("native capture frame header is invalid") } timestamp100ns := binary.BigEndian.Uint64(header[8:16]) duration100ns := binary.BigEndian.Uint64(header[16:24]) - if kind == FrameStatus { - if header[6] != 0 || timestamp100ns != 0 || duration100ns != 0 { - return Frame{}, errors.New("native capture status frame is invalid") - } - } else if duration100ns == 0 || (kind == FramePCM && header[6] != 0) || - ((kind == FrameH264 || kind == FrameVP8) && header[6] > 1) { - return Frame{}, errors.New("native capture media frame is invalid") - } - data := make([]byte, size) - if _, err := io.ReadFull(input, data); err != nil { - return Frame{}, err + width := uint32(binary.BigEndian.Uint16(header[24:26])) + height := uint32(binary.BigEndian.Uint16(header[26:28])) + layer := int(header[7]) + isVideo := kind == FrameH264 || kind == FrameVP8 + isTimed := isVideo || kind == FramePCM || kind == FrameBegin + const maxTime100ns = uint64(1<<63-1) / 100 + if timestamp100ns > maxTime100ns || duration100ns > maxTime100ns || + (isTimed && duration100ns == 0) || (!isTimed && (timestamp100ns != 0 || duration100ns != 0)) || + (isVideo && (header[6] > 1 || layer >= maxOutputs || width < 2 || width > 2560 || + height < 2 || height > 1440 || width%2 != 0 || height%2 != 0)) || + (!isVideo && (header[6] != 0 || width != 0 || height != 0)) || + (kind == FrameLayerUnavailable && layer >= maxOutputs) || + (!isVideo && kind != FrameLayerUnavailable && layer != 0) { + return Frame{}, 0, errors.New("native capture media frame is invalid") } return Frame{ Kind: kind, - KeyFrame: (kind == FrameH264 || kind == FrameVP8) && header[6] == 1, + KeyFrame: isVideo && header[6] == 1, + Layer: layer, + Width: width, + Height: height, Timestamp: time.Duration(timestamp100ns) * 100 * time.Nanosecond, Duration: time.Duration(duration100ns) * 100 * time.Nanosecond, - Data: data, - }, nil + }, size, nil +} + +func validateFrameData(frame Frame) error { + if frame.Kind == FrameLayerUnavailable && !utf8.Valid(frame.Data) { + return errors.New("native unavailable output diagnostic is invalid") + } + if frame.Kind == FrameControl { + for _, value := range frame.Data { + if value < 32 || value > 126 { + return errors.New("native control payload is not ASCII") + } + } + } + return nil +} + +func writeFrame(output io.Writer, frame Frame) error { + if frame.Timestamp < 0 || frame.Duration < 0 || frame.Layer < 0 || frame.Layer > 255 || + frame.Width > 65_535 || frame.Height > 65_535 || len(frame.Data) > maxMediaBytes { + return errors.New("native capture frame is outside its envelope") + } + var header [envelopeHeaderBytes]byte + copy(header[:], "SMED") + header[4], header[5], header[7] = 2, byte(frame.Kind), byte(frame.Layer) + if frame.KeyFrame { + header[6] = 1 + } + binary.BigEndian.PutUint64(header[8:16], uint64(frame.Timestamp/(100*time.Nanosecond))) + binary.BigEndian.PutUint64(header[16:24], uint64(frame.Duration/(100*time.Nanosecond))) + binary.BigEndian.PutUint16(header[24:26], uint16(frame.Width)) + binary.BigEndian.PutUint16(header[26:28], uint16(frame.Height)) + binary.BigEndian.PutUint32(header[28:32], uint32(len(frame.Data))) + if _, _, err := parseFrameHeader(header[:]); err != nil { + return err + } + if err := validateFrameData(frame); err != nil { + return err + } + for _, data := range [][]byte{header[:], frame.Data} { + if len(data) == 0 { + continue + } + n, err := output.Write(data) + if err != nil { + return err + } + if n != len(data) { + return io.ErrShortWrite + } + } + return nil } diff --git a/internal/client/nativecapture/protocol_test.go b/internal/client/nativecapture/protocol_test.go index eb74586..361b7ef 100644 --- a/internal/client/nativecapture/protocol_test.go +++ b/internal/client/nativecapture/protocol_test.go @@ -3,6 +3,9 @@ package nativecapture import ( "bytes" "encoding/binary" + "errors" + "io" + "slices" "testing" "time" ) @@ -11,17 +14,20 @@ func TestReadFrameUsesTheBoundedVersionedEnvelope(t *testing.T) { payload := []byte{0, 0, 0, 1, 0x65} header := make([]byte, envelopeHeaderBytes) copy(header, "SMED") - header[4] = 1 + header[4] = 2 header[5] = byte(FrameH264) header[6] = 1 + header[7] = 1 binary.BigEndian.PutUint64(header[8:16], 123) binary.BigEndian.PutUint64(header[16:24], 333_333) - binary.BigEndian.PutUint32(header[24:], uint32(len(payload))) + binary.BigEndian.PutUint16(header[24:26], 1920) + binary.BigEndian.PutUint16(header[26:28], 1080) + binary.BigEndian.PutUint32(header[28:32], uint32(len(payload))) frame, err := readFrame(bytes.NewReader(append(header, payload...))) if err != nil { t.Fatal(err) } - if frame.Kind != FrameH264 || !frame.KeyFrame || + if frame.Kind != FrameH264 || !frame.KeyFrame || frame.Layer != 1 || frame.Width != 1920 || frame.Height != 1080 || frame.Timestamp != 12_300*time.Nanosecond || frame.Duration != 33_333_300*time.Nanosecond || !bytes.Equal(frame.Data, payload) { @@ -37,16 +43,20 @@ func TestReadFrameUsesTheBoundedVersionedEnvelope(t *testing.T) { func TestReadFrameRejectsInvalidKindsFlagsAndBounds(t *testing.T) { base := make([]byte, envelopeHeaderBytes+1) copy(base, "SMED") - base[4] = 1 + base[4] = 2 base[5] = byte(FrameH264) binary.BigEndian.PutUint64(base[16:24], 1) - binary.BigEndian.PutUint32(base[24:], 1) + binary.BigEndian.PutUint16(base[24:26], 1280) + binary.BigEndian.PutUint16(base[26:28], 720) + binary.BigEndian.PutUint32(base[28:32], 1) for _, mutate := range []func([]byte){ - func(value []byte) { value[4] = 2 }, + func(value []byte) { value[4] = 1 }, func(value []byte) { value[5] = 9 }, func(value []byte) { value[6] = 2 }, - func(value []byte) { value[7] = 1 }, - func(value []byte) { binary.BigEndian.PutUint32(value[24:], maxMediaBytes+1) }, + func(value []byte) { value[7] = maxOutputs }, + func(value []byte) { binary.BigEndian.PutUint16(value[24:26], 1279) }, + func(value []byte) { binary.BigEndian.PutUint64(value[8:16], ^uint64(0)) }, + func(value []byte) { binary.BigEndian.PutUint32(value[28:32], maxMediaBytes+1) }, } { candidate := append([]byte(nil), base...) mutate(candidate) @@ -56,6 +66,157 @@ func TestReadFrameRejectsInvalidKindsFlagsAndBounds(t *testing.T) { } } +func TestMediaEnvelopeAcceptsFourMiBAndRejectsLargerFrames(t *testing.T) { + const limit = 4 * 1024 * 1024 + frame := Frame{Kind: FrameH264, KeyFrame: true, Width: 2560, Height: 1440, + Duration: time.Second / 60, Data: make([]byte, limit)} + var output bytes.Buffer + if err := writeFrame(&output, frame); err != nil { + t.Fatal(err) + } + read, err := readFrame(bytes.NewReader(output.Bytes())) + if err != nil || len(read.Data) != limit || read.Width != frame.Width || read.Height != frame.Height { + t.Fatalf("4 MiB media envelope rejected: bytes=%d err=%v", len(read.Data), err) + } + frame.Data = append(frame.Data, 0) + var rejected bytes.Buffer + if err := writeFrame(&rejected, frame); err == nil || rejected.Len() != 0 { + t.Fatal("oversized media wrote an envelope") + } + header := append([]byte(nil), output.Bytes()[:envelopeHeaderBytes]...) + binary.BigEndian.PutUint32(header[28:32], limit+1) + if _, err := readFrame(bytes.NewReader(header)); err == nil || errors.Is(err, io.EOF) { + t.Fatal("oversized input was not rejected before payload allocation") + } +} + +func TestInputBeginAndUnavailableLayerHaveDistinctEnvelopes(t *testing.T) { + header := make([]byte, envelopeHeaderBytes) + copy(header, "SMED") + header[4], header[5] = 2, byte(FrameBegin) + binary.BigEndian.PutUint64(header[16:24], 333_333) + frame, err := readFrame(bytes.NewReader(header)) + if err != nil || frame.Kind != FrameBegin || len(frame.Data) != 0 || frame.Duration <= 0 { + t.Fatalf("begin = %+v, %v", frame, err) + } + header[5], header[7] = byte(FrameLayerUnavailable), 1 + binary.BigEndian.PutUint64(header[16:24], 0) + payload := []byte("output encoder unavailable") + binary.BigEndian.PutUint32(header[28:32], uint32(len(payload))) + frame, err = readFrame(bytes.NewReader(append(header, payload...))) + if err != nil || frame.Kind != FrameLayerUnavailable || frame.Layer != 1 || !bytes.Equal(frame.Data, payload) { + t.Fatalf("unavailable = %+v, %v", frame, err) + } + binary.BigEndian.PutUint32(header[28:32], 1) + if _, err = readFrame(bytes.NewReader(append(header, 0xff))); err == nil { + t.Fatal("non-UTF8 output diagnostic was accepted") + } + for _, invalid := range []Frame{ + {Kind: FrameControl, Data: bytes.Repeat([]byte{'Q'}, 65)}, + {Kind: FrameControl, Data: []byte("Q\n")}, + {Kind: FrameControl, Data: []byte("Q"), Layer: 1}, + {Kind: FrameControl, Data: []byte("Q"), Timestamp: time.Second}, + {Kind: FrameControl, Data: []byte("Q"), KeyFrame: true}, + } { + var output bytes.Buffer + if err := writeFrame(&output, invalid); err == nil || output.Len() != 0 { + t.Fatal("invalid control wrote a partial envelope") + } + } +} + +func TestOutputProfilesAndControlFramesShareOneBoundedContract(t *testing.T) { + profile := VideoProfile{Width: 854, Height: 480, Framerate: 30, Bitrate: 2_000_000, Preference: "balanced"} + outputs := ScreenShareOutputs(profile) + if !slices.Equal(outputs, []OutputProfile{ + {Width: 426, Height: 240, Framerate: 30, Bitrate: 500_000}, + {Width: 854, Height: 480, Framerate: 30, Bitrate: 2_000_000}, + }) || !outputs[0].Valid() || !outputs[1].Valid() { + t.Fatalf("derived outputs = %+v", outputs) + } + derived := []OutputProfile{{Width: 160, Height: 90, Framerate: 30, Bitrate: 90_000}, {Width: 320, Height: 180, Framerate: 30, Bitrate: 300_000}} + if _, err := appendOutputArguments([]string{"--protocol-v7"}, derived); err != nil { + t.Fatal(err) + } + for _, slots := range [][]OutputProfile{{derived[1], derived[0]}, {derived[0], derived[0]}, + {derived[0], derived[1], derived[0], derived[0], derived[0], derived[0]}} { + if _, err := appendOutputArguments(nil, slots); err != nil { + t.Fatalf("independent output slots rejected: %v", err) + } + } + for _, invalid := range [][]OutputProfile{nil, make([]OutputProfile, maxOutputs+1), {{}}} { + if _, err := appendOutputArguments(nil, invalid); err == nil { + t.Fatal("invalid derived output set was accepted") + } + } + reader, writer := io.Pipe() + done := make(chan error) + close(done) + stream := &Stream{key: writer, done: done, outputs: outputs, cancel: func() {}} + control := make(chan []byte, 1) + go func() { + payload, _ := io.ReadAll(reader) + _ = reader.Close() + control <- payload + }() + for _, command := range []func() error{ + func() error { return stream.SetOutputActive(1, true) }, + func() error { return stream.SetOutputActive(1, true) }, + func() error { return stream.SetOutputActive(0, true) }, + func() error { return stream.SetOutputActive(1, false) }, + func() error { return stream.SetOutputActive(1, false) }, + func() error { return stream.SetOutputBitrate(0, 250_000) }, + func() error { return stream.SetOutputBitrate(0, 250_000) }, + func() error { return stream.RequestKeyFrame(0) }, + func() error { return stream.RequestKeyFrame(-1) }, + func() error { + return stream.WriteFrame(Frame{Kind: FrameVP8, Layer: 0, Width: 160, Height: 90, Timestamp: time.Second, Duration: 33_333_300 * time.Nanosecond, KeyFrame: true, Data: []byte{1, 2, 3}}) + }, + stream.Close, + } { + if err := command(); err != nil { + t.Fatal(err) + } + } + frames := bytes.NewReader(<-control) + var commands []string + var encodedInputs int + for frames.Len() > 0 { + frame, err := readFrame(frames) + if err != nil { + t.Fatal(err) + } + if frame.Kind == FrameControl { + commands = append(commands, string(frame.Data)) + continue + } + if frame.Kind != FrameVP8 || !frame.KeyFrame || frame.Width != 160 || frame.Height != 90 || frame.Timestamp != time.Second || !bytes.Equal(frame.Data, []byte{1, 2, 3}) { + t.Fatalf("encoded input = %+v", frame) + } + encodedInputs++ + } + if !slices.Equal(commands, []string{"A 1 1", "A 0 1", "A 1 0", "B 0 250000", "K 0", "K -1", "Q"}) || encodedInputs != 1 { + t.Fatalf("controls = %q, encoded inputs = %d", commands, encodedInputs) + } + if !errors.Is(stream.RequestKeyFrame(0), io.ErrClosedPipe) || stream.SetOutputActive(2, true) == nil || + stream.SetOutputActive(-1, false) == nil || !errors.Is(stream.SetOutputActive(0, false), io.ErrClosedPipe) { + t.Fatal("closed or invalid controls were accepted") + } +} + +func TestIndependentOutputEnvelopeUsesTheLastBoundedSlot(t *testing.T) { + var output bytes.Buffer + frame := Frame{Kind: FrameVP8, Layer: maxOutputs - 1, Width: 320, Height: 180, + Timestamp: time.Second, Duration: time.Second / 30, Data: []byte{1}} + if err := writeFrame(&output, frame); err != nil { + t.Fatal(err) + } + decoded, err := readFrame(&output) + if err != nil || decoded.Layer != maxOutputs-1 { + t.Fatalf("last output slot = %+v, %v", decoded, err) + } +} + func TestSourceListRejectsPrivateProtocolDrift(t *testing.T) { var targets []CaptureTarget if err := decodeStrictJSON([]byte( @@ -99,8 +260,8 @@ func TestCaptureTargetIdentityAndAudioScope(t *testing.T) { } display := CaptureTarget{Kind: "display", SourceID: "789", Title: "Display 1"} picker := CaptureTarget{Kind: "picker", SourceID: "1", Title: "System picker"} - if !validCaptureTarget(window) || !validCaptureTarget(display) || - !validCaptureTarget(picker) { + if !window.Valid() || !display.Valid() || + !picker.Valid() { t.Fatal("valid capture targets were rejected") } for _, invalid := range []CaptureTarget{ @@ -110,7 +271,7 @@ func TestCaptureTargetIdentityAndAudioScope(t *testing.T) { {Kind: "picker", SourceID: "1", PID: 42, Title: "System picker"}, {Kind: "other", SourceID: "789", Title: "Display 1"}, } { - if validCaptureTarget(invalid) { + if invalid.Valid() { t.Fatalf("invalid capture target was accepted: %+v", invalid) } } diff --git a/internal/client/nativecapture/stream.go b/internal/client/nativecapture/stream.go index c25fc5e..06dacad 100644 --- a/internal/client/nativecapture/stream.go +++ b/internal/client/nativecapture/stream.go @@ -5,8 +5,10 @@ import ( "errors" "fmt" "io" + "log/slog" "os" "os/exec" + "regexp" "strconv" "strings" "sync" @@ -17,6 +19,14 @@ import ( const ( captureStopTimeout = time.Second maxPreviewBytes = 192 * 1024 + maxOutputs = 6 + minOutputBitrate = 1_000 // Codec rate APIs use whole kbps. + maxOutputBitrate = 12_000_000 +) + +var ( + captureStageCode = regexp.MustCompile(`^[a-z][a-z0-9-]{0,63}$`) + captureHRESULT = regexp.MustCompile(`^0x[0-9a-fA-F]{8}$`) ) type audioCaptureState struct { @@ -39,6 +49,15 @@ type VideoOptions struct { EncoderIndex uint32 Profile VideoProfile RestoreToken string + OutputGroups int +} + +type EncodedVideoOptions struct { + Codec string + AdapterIndex uint32 + EncoderIndex uint32 + Preference string + Outputs []OutputProfile } type VideoProfile struct { @@ -49,18 +68,49 @@ type VideoProfile struct { Preference string } +// OutputProfile describes one codec output, not a selectable product preset. +type OutputProfile struct { + Width uint32 `json:"width"` + Height uint32 `json:"height"` + Framerate uint32 `json:"fps"` + Bitrate uint32 `json:"bitrate"` +} + +func (profile OutputProfile) Valid() bool { + return profile.Width >= 2 && profile.Width <= 2560 && profile.Width%2 == 0 && + profile.Height >= 2 && profile.Height <= 1440 && profile.Height%2 == 0 && + profile.Framerate >= 1 && profile.Framerate <= 60 && + profile.Bitrate >= minOutputBitrate && profile.Bitrate <= maxOutputBitrate +} + +// ScreenShareOutputs follows LiveKit 2.22.1's screen-share simulcast construction, +// with even derived dimensions for NV12. The source profile has already passed Valid. +func ScreenShareOutputs(profile VideoProfile) []OutputProfile { + return []OutputProfile{ + { + Width: profile.Width / 4 * 2, Height: profile.Height / 4 * 2, + Framerate: profile.Framerate, Bitrate: max(150_000, profile.Bitrate/4), + }, + { + Width: profile.Width, Height: profile.Height, + Framerate: profile.Framerate, Bitrate: profile.Bitrate, + }, + } +} + func (profile VideoProfile) Valid() bool { validResolution := (profile.Width == 854 && profile.Height == 480) || (profile.Width == 1280 && profile.Height == 720) || (profile.Width == 1920 && profile.Height == 1080) || (profile.Width == 2560 && profile.Height == 1440) - validPreference := profile.Preference == "maintain-resolution" || - profile.Preference == "balanced" || - profile.Preference == "maintain-framerate" return validResolution && profile.Framerate >= 15 && profile.Framerate <= 60 && profile.Bitrate >= 2_000_000 && profile.Bitrate <= 12_000_000 && - validPreference + validVideoPreference(profile.Preference) +} + +func validVideoPreference(preference string) bool { + return preference == "maintain-resolution" || preference == "balanced" || preference == "maintain-framerate" } type Stream struct { @@ -69,6 +119,11 @@ type Stream struct { key io.WriteCloser done chan error + outputs []OutputProfile + active [maxOutputs]bool + bitrates [maxOutputs]uint32 + controlMu sync.Mutex + closed bool closeOnce sync.Once } @@ -88,7 +143,7 @@ func ListSources(parent context.Context, executable string) ([]CaptureTarget, er return nil, errors.New("native capture source list is invalid") } for _, target := range targets { - if !validCaptureTarget(target) { + if !target.Valid() { return nil, errors.New("native capture target is invalid") } } @@ -96,7 +151,7 @@ func ListSources(parent context.Context, executable string) ([]CaptureTarget, er } func PreviewSource(parent context.Context, executable string, target CaptureTarget) ([]byte, error) { - if !validCaptureTarget(target) { + if !target.Valid() { return nil, errors.New("native preview target is invalid") } ctx, cancel := context.WithTimeout(parent, probeTimeout) @@ -123,8 +178,9 @@ func PreviewSource(parent context.Context, executable string, target CaptureTarg } func StartVideo(parent context.Context, executable string, options VideoOptions) (*Stream, error) { - if !validCaptureTarget(options.Target) || !options.Profile.Valid() || + if !options.Target.Valid() || !options.Profile.Valid() || (options.Codec != "auto" && options.Codec != "h264" && options.Codec != "vp8") || + options.OutputGroups < 0 || options.OutputGroups > maxOutputs-2 || len(options.RestoreToken) > 4096 || !utf8.ValidString(options.RestoreToken) || strings.ContainsRune(options.RestoreToken, 0) { return nil, errors.New("native video target is invalid") @@ -133,7 +189,7 @@ func StartVideo(parent context.Context, executable string, options VideoOptions) if options.RestoreToken != "" { environment = []string{"SCREENER_XDP_RESTORE_TOKEN=" + options.RestoreToken} } - return startStreamWithEnvironment(parent, executable, []string{ + arguments := []string{ "--capture-video", options.Target.Kind, options.Target.SourceID, @@ -155,12 +211,62 @@ func StartVideo(parent context.Context, executable string, options VideoOptions) options.Profile.Preference, "--codec", options.Codec, - "--protocol-v4", - }, environment) + "--protocol-v7", + } + outputs := ScreenShareOutputs(options.Profile) + for range options.OutputGroups { + outputs = append(outputs, outputs[0]) + } + arguments, err := appendOutputArguments(arguments, outputs) + if err != nil { + return nil, err + } + stream, err := startStreamWithEnvironment(parent, executable, arguments, environment) + if err != nil { + return nil, err + } + stream.outputs = outputs + stream.active[0], stream.active[1] = true, true + return stream, nil +} + +func StartEncodedVideo(parent context.Context, executable string, options EncodedVideoOptions) (*Stream, error) { + if (options.Codec != "h264" && options.Codec != "vp8") || !validVideoPreference(options.Preference) { + return nil, errors.New("native encoded video options are invalid") + } + arguments, err := appendOutputArguments([]string{ + "--encoded-video", "--codec", options.Codec, + "--adapter-index", strconv.FormatUint(uint64(options.AdapterIndex), 10), + "--mft-index", strconv.FormatUint(uint64(options.EncoderIndex), 10), + "--preference", options.Preference, "--protocol-v7", + }, options.Outputs) + if err != nil { + return nil, err + } + stream, err := startStream(parent, executable, arguments) + if err != nil { + return nil, err + } + stream.outputs = append([]OutputProfile(nil), options.Outputs...) + return stream, nil +} + +func appendOutputArguments(arguments []string, outputs []OutputProfile) ([]string, error) { + if len(outputs) < 1 || len(outputs) > maxOutputs { + return nil, errors.New("native output count is invalid") + } + for _, output := range outputs { + if !output.Valid() { + return nil, errors.New("native output profile is invalid") + } + arguments = append(arguments, "--output", strconv.FormatUint(uint64(output.Width), 10), + strconv.FormatUint(uint64(output.Height), 10), strconv.FormatUint(uint64(output.Framerate), 10), strconv.FormatUint(uint64(output.Bitrate), 10)) + } + return arguments, nil } func StartAudio(parent context.Context, executable string, target CaptureTarget) (*Stream, error) { - if !validCaptureTarget(target) { + if !target.Valid() { return nil, errors.New("native audio target is invalid") } return startAudioStream(parent, executable, []string{ @@ -212,7 +318,7 @@ func startAudioStream(parent context.Context, executable string, arguments []str } } -func validCaptureTarget(target CaptureTarget) bool { +func (target CaptureTarget) Valid() bool { if !positiveDecimal(target.SourceID) || len(target.Title) == 0 || len(target.Title) > 4096 { return false @@ -289,7 +395,8 @@ func startStreamWithEnvironment( cancel() return nil, err } - command.Stderr = &boundedBuffer{limit: maxProbeErrorBytes} + stderr := &boundedBuffer{limit: maxProbeErrorBytes} + command.Stderr = stderr hideWindow(command) if err = command.Start(); err != nil { cancel() @@ -302,19 +409,120 @@ func startStreamWithEnvironment( done: make(chan error, 1), } go func() { - stream.done <- command.Wait() + waitErr := command.Wait() + logCaptureFailure(ctx, waitErr, command.ProcessState.ExitCode(), stderr.Bytes()) + stream.done <- waitErr close(stream.done) }() return stream, nil } +func logCaptureFailure(ctx context.Context, err error, exitCode int, stderr []byte) { + if err == nil || !slog.Default().Enabled(ctx, slog.LevelDebug) { + return + } + stage, hresult := "", "" + for _, line := range strings.Split(string(stderr), "\n") { + key, value, _ := strings.Cut(strings.TrimSuffix(line, "\r"), "=") + switch { + case key == "stage" && stage == "" && captureStageCode.MatchString(value): + stage = value + case key == "hresult" && hresult == "" && captureHRESULT.MatchString(value): + hresult = strings.ToLower(value) + } + } + canceled := ctx.Err() != nil + // A real child failure may race owner cancellation. Keep its fixed codes, + // while ordinary cancellation without a reported failure stays silent. + if canceled && stage == "" && hresult == "" { + return + } + slog.DebugContext(ctx, "screener-client", "event", "capture-process-failed", + "exitCode", exitCode, "stage", stage, "hresult", hresult, "canceled", canceled) +} + func (stream *Stream) Read() (Frame, error) { return readFrame(stream.input) } -func (stream *Stream) RequestKeyFrame() error { - _, err := stream.key.Write([]byte{'K'}) - return err +// WriteFrame shares the existing stdin lock with control. Only encoded video +// crosses this input boundary; capture status and PCM remain producer outputs. +func (stream *Stream) WriteFrame(frame Frame) error { + if frame.Kind != FrameH264 && frame.Kind != FrameVP8 { + return errors.New("native encoded input is not video") + } + stream.controlMu.Lock() + defer stream.controlMu.Unlock() + if stream.closed { + return io.ErrClosedPipe + } + return writeFrame(stream.key, frame) +} + +func (stream *Stream) Outputs() []OutputProfile { + return append([]OutputProfile(nil), stream.outputs...) +} + +func (stream *Stream) RequestKeyFrame(layer int) error { + if layer < -1 || layer >= len(stream.outputs) || len(stream.outputs) == 0 { + return errors.New("native recovery output is invalid") + } + return stream.writeControl("K " + strconv.Itoa(layer)) +} + +func (stream *Stream) SetOutputActive(layer int, active bool) error { + if layer < 0 || layer >= len(stream.outputs) { + return errors.New("native active output is invalid") + } + stream.controlMu.Lock() + defer stream.controlMu.Unlock() + if stream.closed { + return io.ErrClosedPipe + } + if stream.active[layer] == active { + return nil + } + value := "0" + if active { + value = "1" + } + if err := stream.writeControlLocked("A " + strconv.Itoa(layer) + " " + value); err != nil { + return err + } + stream.active[layer] = active + return nil +} + +func (stream *Stream) SetOutputBitrate(layer int, bitrate uint32) error { + if layer < 0 || layer >= len(stream.outputs) || bitrate < minOutputBitrate || bitrate > stream.outputs[layer].Bitrate { + return errors.New("native output bitrate is invalid") + } + stream.controlMu.Lock() + defer stream.controlMu.Unlock() + if stream.closed { + return io.ErrClosedPipe + } + if stream.bitrates[layer] == bitrate { + return nil + } + if err := stream.writeControlLocked("B " + strconv.Itoa(layer) + " " + strconv.FormatUint(uint64(bitrate), 10)); err != nil { + return err + } + stream.bitrates[layer] = bitrate + return nil +} + +func (stream *Stream) writeControl(command string) error { + stream.controlMu.Lock() + defer stream.controlMu.Unlock() + return stream.writeControlLocked(command) +} + +func (stream *Stream) writeControlLocked(command string) error { + if stream.closed { + return io.ErrClosedPipe + } + return writeFrame(stream.key, Frame{Kind: FrameControl, Data: []byte(command)}) } func (stream *Stream) Done() <-chan error { @@ -322,17 +530,17 @@ func (stream *Stream) Done() <-chan error { } func (stream *Stream) Close() error { + // Start the existing stop budget before waiting for a possibly blocked control write. + timer := time.AfterFunc(captureStopTimeout, stream.cancel) + defer timer.Stop() stream.closeOnce.Do(func() { - _, _ = stream.key.Write([]byte{'Q'}) + stream.controlMu.Lock() + defer stream.controlMu.Unlock() + stream.closed = true + _ = writeFrame(stream.key, Frame{Kind: FrameControl, Data: []byte("Q")}) _ = stream.key.Close() }) - select { - case <-stream.done: - stream.cancel() - return nil - case <-time.After(captureStopTimeout): - stream.cancel() - <-stream.done - return nil - } + <-stream.done + stream.cancel() + return nil } diff --git a/internal/client/nativecapture/stream_test.go b/internal/client/nativecapture/stream_test.go new file mode 100644 index 0000000..e62eae5 --- /dev/null +++ b/internal/client/nativecapture/stream_test.go @@ -0,0 +1,79 @@ +package nativecapture + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "log/slog" + "strings" + "testing" +) + +func TestVideoOutputGroupCountIsBoundedBeforeProcessStartup(t *testing.T) { + options := VideoOptions{ + Target: CaptureTarget{Kind: "display", SourceID: "1", Title: "Display"}, + Codec: "vp8", + Profile: VideoProfile{Width: 1280, Height: 720, Framerate: 30, + Bitrate: 3_000_000, Preference: "balanced"}, + } + for _, groups := range []int{-1, maxOutputs - 1} { + options.OutputGroups = groups + if _, err := StartVideo(t.Context(), "", options); err == nil || err.Error() != "native video target is invalid" { + t.Fatalf("group count %d reached process startup: %v", groups, err) + } + } +} + +func TestCaptureFailureDebugUsesOnlyFixedFields(t *testing.T) { + previous := slog.Default() + t.Cleanup(func() { slog.SetDefault(previous) }) + for _, check := range []struct { + name, stderr, stage, hresult string + debug, canceled, success, logged bool + }{ + {name: "closed", stderr: "result=window-capture-failed\r\nstage=capture-closed\r\ndetail=private window title\r\n", stage: "capture-closed", debug: true, logged: true}, + {name: "device", stderr: "stage=windows-runtime\nhresult=0x887A0005\ndetail=C:\\private\\capture.exe", stage: "windows-runtime", hresult: "0x887a0005", debug: true, logged: true}, + {name: "malformed", stderr: "stage=C:\\private\\capture.exe\nhresult=0x887a0005 private\n", debug: true, logged: true}, + {name: "oversized-stage", stderr: "stage=" + strings.Repeat("a", 65), debug: true, logged: true}, + {name: "private-detail", stderr: "detail=private window title\n", debug: true, logged: true}, + {name: "disabled", stderr: "stage=capture-closed\n", debug: false}, + {name: "clean", stderr: "stage=capture-closed\n", debug: true, success: true}, + {name: "canceled", stderr: "detail=private cancellation\n", debug: true, canceled: true}, + {name: "failure-before-cancel", stderr: "stage=capture-closed\n", stage: "capture-closed", debug: true, canceled: true, logged: true}, + } { + t.Run(check.name, func(t *testing.T) { + var output bytes.Buffer + level := slog.LevelInfo + if check.debug { + level = slog.LevelDebug + } + slog.SetDefault(slog.New(slog.NewJSONHandler(&output, &slog.HandlerOptions{Level: level}))) + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + if check.canceled { + cancel() + } + var failure error + if !check.success { + failure = errors.New("private executable path") + } + logCaptureFailure(ctx, failure, 2, []byte(check.stderr)) + if !check.logged { + if output.Len() != 0 { + t.Fatalf("unexpected diagnostic: %s", output.String()) + } + return + } + var record map[string]any + if err := json.Unmarshal(output.Bytes(), &record); err != nil { + t.Fatal(err) + } + if len(record) != 8 || record["event"] != "capture-process-failed" || record["msg"] != "screener-client" || + record["level"] != "DEBUG" || record["exitCode"] != float64(2) || record["stage"] != check.stage || + record["hresult"] != check.hresult || record["canceled"] != check.canceled || strings.Contains(output.String(), "private") { + t.Fatalf("diagnostic leaked or changed fields: %s", output.String()) + } + }) + } +} diff --git a/internal/client/nativecontrol/publication.go b/internal/client/nativecontrol/publication.go new file mode 100644 index 0000000..0911734 --- /dev/null +++ b/internal/client/nativecontrol/publication.go @@ -0,0 +1,83 @@ +package nativecontrol + +import ( + "encoding/json" + "errors" + + "github.com/pion/webrtc/v4" +) + +func (session *Session) handlePublication(envelope requestEnvelope, payload []byte) (any, error) { + var identity publicationIdentity + if err := json.Unmarshal(payload, &identity); err != nil || + !validIdentities(identity.ShareID, identity.PublicationGeneration, identity.ConnectionID) { + return nil, errors.New("native publication identity is invalid") + } + host := session.current(identity.ShareID) + if host == nil { + return nil, errors.New("native share does not exist") + } + result := publicationResponse{ShareID: identity.ShareID, + PublicationGeneration: identity.PublicationGeneration, ConnectionID: identity.ConnectionID} + if envelope.Type == "prepare-publication" { + var request preparePublicationRequest + if err := decodeStrict(payload, &request); err != nil { + return nil, err + } + servers, err := pionICEServers(request.ICEServers) + if err != nil { + return nil, err + } + offer, media, err := host.PreparePublication(identity.PublicationGeneration, identity.ConnectionID, servers) + if err != nil { + return operationFailure(envelope), nil + } + result.responseEnvelope, result.SDP, result.Media = response(envelope, "publication-offer"), offer.SDP, media + return result, nil + } + if envelope.Type == "close-publication" { + if err := decodeStrict(payload, &identity); err != nil { + return nil, err + } + host.ClosePublication(identity.PublicationGeneration, identity.ConnectionID) + return response(envelope, "publication-closed"), nil + } + publication, err := host.Publication(identity.PublicationGeneration, identity.ConnectionID) + if err != nil { + return operationFailure(envelope), nil + } + accepted := "" + switch envelope.Type { + case "publication-media": + if err = decodeStrict(payload, &identity); err != nil { + return nil, err + } + result.responseEnvelope, result.Media = response(envelope, "publication-media"), publication.Media() + return result, nil + case "publication-answer": + var request publicationAnswerRequest + if err = decodeStrict(payload, &request); err != nil || len(request.SDP) == 0 || len(request.SDP) > maxSDPBytes { + return nil, errors.New("native publication answer is invalid") + } + err = publication.SetAnswer(webrtc.SessionDescription{Type: webrtc.SDPTypeAnswer, SDP: request.SDP}) + accepted = "publication-answer-accepted" + case "publication-candidate": + var request publicationCandidateRequest + if err = decodeStrict(payload, &request); err != nil || !validCandidate(request.Candidate) { + return nil, errors.New("native publication candidate is invalid") + } + err = publication.AddRemoteCandidate(request.Candidate) + accepted = "publication-candidate-accepted" + case "publication-layers": + var request publicationLayersRequest + if err = decodeStrict(payload, &request); err != nil || request.ActiveCount == nil || *request.ActiveCount < 0 || *request.ActiveCount > 3 { + return nil, errors.New("native publication layer demand is invalid") + } + err = publication.SetActiveCount(*request.ActiveCount) + accepted = "publication-layers-accepted" + } + if err != nil { + return operationFailure(envelope), nil + } + return response(envelope, accepted), nil +} diff --git a/internal/client/nativecontrol/session.go b/internal/client/nativecontrol/session.go index 8841fcd..aa3aaa5 100644 --- a/internal/client/nativecontrol/session.go +++ b/internal/client/nativecontrol/session.go @@ -6,7 +6,9 @@ import ( "encoding/base64" "encoding/json" "errors" + "fmt" "io" + "log/slog" "net/url" "regexp" "strconv" @@ -15,6 +17,7 @@ import ( "unicode" "github.com/TNTcraftHIM/Screener/internal/client/loopback" + "github.com/TNTcraftHIM/Screener/internal/client/mediaedge" "github.com/TNTcraftHIM/Screener/internal/client/nativecapture" "github.com/TNTcraftHIM/Screener/internal/client/nativehost" "github.com/TNTcraftHIM/Screener/internal/client/nativeviewer" @@ -42,10 +45,11 @@ type Session struct { hostEvents chan nativehost.Event viewerEvents chan nativeviewer.Event - mu sync.Mutex - host *nativehost.Session - viewer *nativeviewer.Session - closed bool + mu sync.Mutex + host *nativehost.Session + viewer *nativeviewer.Session + closed bool + updateDone chan struct{} } type outboundMediaSession interface { @@ -84,6 +88,8 @@ func (session *Session) Handle(ctx context.Context, payload []byte) (any, error) return nil, err } switch envelope.Type { + case "prepare-publication", "publication-media", "publication-answer", "publication-candidate", "publication-layers", "close-publication": + return session.handlePublication(envelope, payload) case "capture-options": var request captureOptionsRequest if err := decodeStrict(payload, &request); err != nil || request.Type != envelope.Type { @@ -100,7 +106,7 @@ func (session *Session) Handle(ctx context.Context, payload []byte) (any, error) } targets, err := nativecapture.ListSources(session.ctx, session.captureProcess) if err != nil { - return nil, err + return operationFailure(envelope), nil } return sourceListResponse{ responseEnvelope: response(envelope, "source-list"), @@ -146,37 +152,46 @@ func (session *Session) Handle(ctx context.Context, payload []byte) (any, error) !validQualitySettings(request.Profile) { return nil, errors.New("native update-share request is invalid") } - host := session.current(request.ShareID) - if host == nil { + profile := nativeQualityProfile(request.Profile) + var err error + if host := session.current(request.ShareID); host != nil { + return session.updateHost(host, envelope, request.ShareID, profile), nil + } else if viewer := session.currentViewer(request.ShareID); viewer != nil { + err = viewer.UpdateProfile(profile.Video) + } else { return nil, errors.New("native share does not exist") } - if err := host.UpdateProfile(nativeQualityProfile(request.Profile)); err != nil { - return nil, err - } - return shareUpdatedResponse{ - responseEnvelope: response(envelope, "share-updated"), - ShareID: request.ShareID, - }, nil + return session.shareUpdateResult(envelope, request.ShareID, profile, err), nil case "replace-share-source": var request replaceShareSourceRequest if err := decodeStrict(payload, &request); err != nil || request.Type != envelope.Type || - !validIdentities(request.ShareID) { + !validIdentities(request.ShareID) || !request.Source.Valid() { return nil, errors.New("native replace-share-source request is invalid") } host := session.current(request.ShareID) if host == nil { return nil, errors.New("native share does not exist") } + session.mu.Lock() + updating := session.updateDone != nil + session.mu.Unlock() + if updating { + return operationFailure(envelope), nil + } audio := request.Audio && session.capabilities.Summary().AudioFor( request.Source.Kind, ) - if err := host.ReplaceSource(ctx, nativecapture.VideoOptions{ + err := host.ReplaceSource(ctx, nativecapture.VideoOptions{ Target: request.Source, AdapterIndex: request.AdapterIndex, EncoderIndex: request.EncoderIndex, - }, audio); err != nil { - return nil, err + }, audio) + if slog.Default().Enabled(session.ctx, slog.LevelDebug) { + slog.Debug("screener-client", "event", "share-source-replace", "failed", err != nil, "errorType", fmt.Sprintf("%T", err)) + } + if err != nil { + return operationFailure(envelope), nil } return shareSourceReplacedResponse{ responseEnvelope: response(envelope, "share-source-replaced"), @@ -296,7 +311,11 @@ func (session *Session) Handle(ctx context.Context, payload []byte) (any, error) !identityPattern.MatchString(request.ShareID) { return nil, errors.New("native stop-share request is invalid") } - if err := session.stopShare(request.ShareID); err != nil { + err := session.stopShare(request.ShareID) + if slog.Default().Enabled(session.ctx, slog.LevelDebug) { + slog.Debug("screener-client", "event", "share-stop", "failed", err != nil, "errorType", fmt.Sprintf("%T", err)) + } + if err != nil { return nil, err } return response(envelope, "share-stopped"), nil @@ -325,6 +344,7 @@ func (session *Session) Close() error { return nil } session.closed = true + updateDone := session.updateDone host := session.host viewer := session.viewer session.host = nil @@ -337,21 +357,92 @@ func (session *Session) Close() error { if viewer != nil { _ = viewer.Close() } + if updateDone != nil { + <-updateDone + } return nil } +func (session *Session) updateHost(host *nativehost.Session, request requestEnvelope, shareID string, profile nativehost.QualityProfile) any { + session.mu.Lock() + if session.closed || session.host != host || session.updateDone != nil { + session.mu.Unlock() + return operationFailure(request) + } + done := make(chan struct{}) + session.updateDone = done + session.mu.Unlock() + go func() { + err := host.UpdateProfile(profile) + session.mu.Lock() + current := !session.closed && session.host == host + if !current && err == nil { + err = errors.New("native share stopped during quality update") + } + result := session.shareUpdateResult(request, shareID, profile, err) + close(done) + if session.closed { + session.updateDone = nil + session.mu.Unlock() + return + } + // Release admission before an enqueued ACK can trigger the next update. + select { + case session.events <- result: + session.updateDone = nil + session.mu.Unlock() + return + default: + session.mu.Unlock() + } + // Backpressure retains one completion but never holds the stop mutex. + session.emit(result) + session.mu.Lock() + if session.updateDone == done { + session.updateDone = nil + } + session.mu.Unlock() + }() + return nil +} + +func (session *Session) shareUpdateResult(request requestEnvelope, shareID string, profile nativehost.QualityProfile, err error) any { + if slog.Default().Enabled(session.ctx, slog.LevelDebug) { + slog.Debug("screener-client", "event", "share-update", "failed", err != nil, "errorType", fmt.Sprintf("%T", err), + "width", profile.Video.Width, "height", profile.Video.Height, "fps", profile.Video.Framerate, + "bitrate", profile.Video.Bitrate, "audioBitrate", profile.AudioBitrate) + } + if err != nil { + return operationFailure(request) + } + return shareUpdatedResponse{responseEnvelope: response(request, "share-updated"), ShareID: shareID} +} + func (session *Session) startShare( ctx context.Context, envelope requestEnvelope, request startShareRequest, -) (any, error) { +) (result any, returnedErr error) { + profile := nativeQualityProfile(request.Profile) + codec := request.Codec + defer func() { + if slog.Default().Enabled(session.ctx, slog.LevelDebug) { + _, rejected := result.(requestFailedResponse) + slog.Debug("screener-client", "event", "share-start", "failed", returnedErr != nil || rejected, "errorType", fmt.Sprintf("%T", returnedErr), + "codec", codec, "width", profile.Video.Width, "height", profile.Video.Height, + "fps", profile.Video.Framerate, "bitrate", profile.Video.Bitrate) + } + }() session.mu.Lock() + if session.updateDone != nil { + session.mu.Unlock() + return operationFailure(envelope), nil + } if session.closed || session.host != nil || session.viewer != nil { session.mu.Unlock() return nil, errors.New("native share is already active") } session.mu.Unlock() - profile := nativeQualityProfile(request.Profile) host, err := nativehost.Start(ctx, nativehost.Options{ ShareID: request.ShareID, CaptureProcess: session.captureProcess, @@ -381,12 +472,13 @@ func (session *Session) startShare( } session.host = host session.mu.Unlock() + codec = host.Codec() go session.watchHost(host) return shareStartedResponse{ responseEnvelope: response(envelope, "share-started"), ShareID: request.ShareID, Audio: host.HasAudio(), - Codec: host.Codec(), + Codec: codec, }, nil } @@ -525,8 +617,13 @@ func (session *Session) stopShare(shareID string) error { return errors.New("native share does not exist") } session.host = nil + updateDone := session.updateDone session.mu.Unlock() - return host.Close() + err := host.Close() + if updateDone != nil { + <-updateDone + } + return err } func (session *Session) current(shareID string) *nativehost.Session { @@ -559,6 +656,7 @@ func (session *Session) ensureViewer( viewer, err := nativeviewer.Start(session.ctx, nativeviewer.Options{ ShareID: shareID, EdgeCapacity: edgeCapacity, PortMapping: session.portMapping, Events: session.viewerEvents, + Relay: &mediaedge.RelayOptions{CaptureProcess: session.captureProcess, Capabilities: session.capabilities}, }) if err != nil { return nil, err @@ -690,15 +788,18 @@ func (session *Session) emit(event any) { func eventMessage(event nativehost.Event) any { base := eventEnvelope{ - Version: loopback.ProtocolVersion, - Type: event.Type, - ShareID: event.ShareID, - ConnectionID: event.ConnectionID, + Version: loopback.ProtocolVersion, + Type: event.Type, + ShareID: event.ShareID, + ConnectionID: event.ConnectionID, + PublicationGeneration: event.PublicationGeneration, } switch event.Type { - case "edge-candidate": + case "publication-quality": + return publicationQualityEvent{eventEnvelope: base, PublicationQualitySample: *event.PublicationQuality} + case "edge-candidate", "publication-candidate": return edgeCandidateEvent{eventEnvelope: base, Candidate: event.Candidate} - case "edge-state": + case "edge-state", "publication-state": return edgeStateEvent{eventEnvelope: base, State: event.State} case "edge-path": return edgePathEvent{ @@ -827,3 +928,10 @@ func response(request requestEnvelope, responseType string) responseEnvelope { Type: responseType, } } + +func operationFailure(request requestEnvelope) requestFailedResponse { + return requestFailedResponse{ + responseEnvelope: response(request, "request-failed"), + Code: "operation-failed", + } +} diff --git a/internal/client/nativecontrol/session_test.go b/internal/client/nativecontrol/session_test.go index 71cce83..abb3ea5 100644 --- a/internal/client/nativecontrol/session_test.go +++ b/internal/client/nativecontrol/session_test.go @@ -1,12 +1,249 @@ package nativecontrol import ( + "context" + "encoding/binary" + "encoding/json" + "io" + "os" + "path/filepath" + "strconv" + "strings" "testing" + "time" "github.com/TNTcraftHIM/Screener/internal/client/loopback" "github.com/TNTcraftHIM/Screener/internal/client/nativecapture" + "github.com/TNTcraftHIM/Screener/internal/client/nativehost" ) +func TestMain(tests *testing.M) { + if directory := os.Getenv("SCREENER_QUIET_CAPTURE_FIXTURE"); directory != "" { + runQuietCaptureFixture(directory) + os.Exit(0) + } + os.Exit(tests.Run()) +} + +func runQuietCaptureFixture(directory string) { + var state nativehost.CaptureState + state.State, state.Codec = "starting", "vp8" + var adapter uint32 + state.AdapterIndex = &adapter + state.AdapterName, state.AdapterIdentity, state.EncoderName, state.EncoderIdentity = "fixture", "fixture", "fixture", "fixture" + number := func(value string) uint32 { parsed, _ := strconv.ParseUint(value, 10, 32); return uint32(parsed) } + for index := 1; index < len(os.Args)-1; index++ { + switch os.Args[index] { + case "--width": + state.Width = number(os.Args[index+1]) + case "--height": + state.Height = number(os.Args[index+1]) + case "--fps": + state.FPS = number(os.Args[index+1]) + case "--output": + state.Outputs = append(state.Outputs, nativecapture.OutputProfile{Width: number(os.Args[index+1]), + Height: number(os.Args[index+2]), Framerate: number(os.Args[index+3]), Bitrate: number(os.Args[index+4])}) + } + } + write := func(kind nativecapture.FrameKind, payload []byte, timestamp uint64) { + var header [32]byte + copy(header[:], "SMED") + header[4], header[5] = 2, byte(kind) + if kind == nativecapture.FrameBegin { + binary.BigEndian.PutUint64(header[8:16], timestamp) + binary.BigEndian.PutUint64(header[16:24], 333_333) + } + binary.BigEndian.PutUint32(header[28:32], uint32(len(payload))) + _, _ = os.Stdout.Write(header[:]) + _, _ = os.Stdout.Write(payload) + } + status := func() { payload, _ := json.Marshal(state); write(nativecapture.FrameStatus, payload, 0) } + status() + closed := make(chan struct{}) + go func() { _, _ = io.Copy(io.Discard, os.Stdin); close(closed) }() + if state.Width == 1280 { + _ = os.WriteFile(filepath.Join(directory, "prepared"), nil, 0600) + ticker := time.NewTicker(20 * time.Millisecond) + defer ticker.Stop() + var input uint64 + for { + select { + case <-closed: + return + case <-ticker.C: + value, _ := os.ReadFile(filepath.Join(directory, "release")) + if string(value) == "active" { + write(nativecapture.FrameBegin, nil, input+333_333) + state.State = "active" + status() + <-closed + return + } + if string(value) == "input" { + input += 333_333 + write(nativecapture.FrameBegin, nil, input) + } + } + } + } + state.State = "active" + status() + <-closed +} + +func TestQuietHostUpdatePreservesControlAndCancelsCleanly(t *testing.T) { + for _, outcome := range []string{"complete", "stop", "disconnect", "first-input-timeout", "backpressure-stop"} { + t.Run(outcome, func(t *testing.T) { + directory := t.TempDir() + t.Setenv("SCREENER_QUIET_CAPTURE_FIXTURE", directory) + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + session := New(executable, nativecapture.Capabilities{SoftwareVP8: true}, false) + t.Cleanup(func() { _ = session.Close() }) + handle := func(payload string) any { + t.Helper() + value, err := session.Handle(t.Context(), []byte(payload)) + if err != nil { + t.Fatal(err) + } + return value + } + const start = `{"version":9,"id":"request_start","type":"start-share","shareId":"share_123456","source":{"kind":"display","sourceId":"1","title":"Fixture"},"audio":false,"adapterIndex":0,"encoderIndex":0,"edgeCapacity":1,"codec":"vp8","profile":{"resolution":"1080p","maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced"}}` + handle(start) + const update = `{"version":9,"id":"request_update","type":"update-share","shareId":"share_123456","profile":{"resolution":"720p","maxFramerate":30,"maxBitrate":3000000,"degradationPreference":"balanced"}}` + if value := handle(update); value != nil { + t.Fatalf("Host update did not defer: %#v", value) + } + session.mu.Lock() + done := session.updateDone + session.mu.Unlock() + deadline := time.Now().Add(2 * time.Second) + for { + if _, err = os.Stat(filepath.Join(directory, "prepared")); err == nil { + break + } + if time.Now().After(deadline) { + t.Fatal("replacement did not reach its starting state") + } + time.Sleep(10 * time.Millisecond) + } + for _, payload := range []string{ + strings.Replace(update, "request_update", "request_duplicate", 1), + `{"version":9,"id":"request_replace","type":"replace-share-source","shareId":"share_123456","source":{"kind":"display","sourceId":"1","title":"Fixture"},"audio":false,"adapterIndex":0,"encoderIndex":0}`, + } { + if _, ok := handle(payload).(requestFailedResponse); !ok { + t.Fatal("conflicting capture mutation was not rejected") + } + } + for _, payload := range []string{ + `{"version":9,"id":"request_pause","type":"pause-share","shareId":"share_123456","paused":true}`, + `{"version":9,"id":"request_ice","type":"edge-candidate","shareId":"share_123456","connectionId":"retired_edge","candidate":null}`, + } { + started := time.Now() + handle(payload) + if time.Since(started) > time.Second { + t.Fatal("quality preparation blocked control") + } + } + switch outcome { + case "complete": + select { + case <-done: + t.Fatal("quiet source update ended before a frame arrived") + case <-time.After(5200 * time.Millisecond): + } + if err = os.WriteFile(filepath.Join(directory, "release"), []byte("active"), 0600); err != nil { + t.Fatal(err) + } + case "first-input-timeout": + if err = os.WriteFile(filepath.Join(directory, "release"), []byte("input"), 0600); err != nil { + t.Fatal(err) + } + case "backpressure-stop": + fill: + for { + select { + case session.events <- struct{}{}: + default: + break fill + } + } + if err = os.WriteFile(filepath.Join(directory, "release"), []byte("active"), 0600); err != nil { + t.Fatal(err) + } + case "stop": + handle(`{"version":9,"id":"request_stop","type":"stop-share","shareId":"share_123456"}`) + if _, ok := handle(strings.ReplaceAll(start, "share_123456", "share_restart")).(shareStartedResponse); !ok { + t.Fatal("stopped update blocked a new share") + } + case "disconnect": + if err = session.Close(); err != nil { + t.Fatal(err) + } + } + select { + case <-done: + case <-time.After(6 * time.Second): + t.Fatal("update did not finish after input or cancellation") + } + if outcome == "disconnect" { + return + } + if outcome == "backpressure-stop" { + stopped := make(chan any, 1) + go func() { + value, err := session.Handle(t.Context(), []byte(`{"version":9,"id":"request_stop","type":"stop-share","shareId":"share_123456"}`)) + if err != nil { + stopped <- err + } else { + stopped <- value + } + }() + select { + case value := <-stopped: + if _, ok := value.(responseEnvelope); !ok { + t.Fatalf("backpressured stop failed: %#v", value) + } + case <-time.After(2 * time.Second): + t.Fatal("completion backpressure blocked stop") + } + return + } + ctx, cancel := context.WithTimeout(t.Context(), time.Second) + defer cancel() + responseFor := func(id string) any { + for { + select { + case value := <-session.Events(): + if response, ok := value.(shareUpdatedResponse); ok && response.ID == id { + return value + } + if response, ok := value.(requestFailedResponse); ok && response.ID == id { + return value + } + case <-ctx.Done(): + t.Fatal("deferred update lost its response") + } + } + } + _, success := responseFor("request_update").(shareUpdatedResponse) + if success != (outcome == "complete") { + t.Fatal("deferred response did not match capture outcome") + } + if outcome == "complete" { + if value := handle(strings.Replace(update, "request_update", "request_followup", 1)); value != nil { + t.Fatal("acknowledged update still blocked its successor") + } + if _, ok := responseFor("request_followup").(shareUpdatedResponse); !ok { + t.Fatal("later update could not complete") + } + } + }) + } +} + func TestSTUNURLsUseTheCurrentBoundedWire(t *testing.T) { servers, err := pionICEServers([]iceServer{{URLs: []string{ "stun:example.test:3478", @@ -31,7 +268,7 @@ func TestControlMessagesRejectUnknownFieldsAndStaleVersions(t *testing.T) { session := New("missing-capture-process", nativecapture.Capabilities{}, false) t.Cleanup(func() { _ = session.Close() }) for _, payload := range []string{ - `{"version":8,"id":"request_sources","type":"list-sources","extra":true}`, + `{"version":9,"id":"request_sources","type":"list-sources","extra":true}`, `{"version":4,"id":"request_sources","type":"list-sources"}`, `{"version":5,"id":"short","type":"stop-share","shareId":"share_123456"}`, } { @@ -51,16 +288,23 @@ func TestResponseKeepsTheRequestIdentity(t *testing.T) { value.Type != "share-stopped" { t.Fatalf("response = %+v", value) } + failed := operationFailure(requestEnvelope{ + Version: loopback.ProtocolVersion, ID: "request_update", Type: "update-share", + }) + if failed.Version != loopback.ProtocolVersion || failed.ID != "request_update" || + failed.Type != "request-failed" || failed.Code != "operation-failed" { + t.Fatalf("operation failure lost its request identity: %+v", failed) + } } func TestPrepareLocalEdgeOwnsOneStrictRequestShape(t *testing.T) { session := New("missing-capture-process", nativecapture.Capabilities{}, false) t.Cleanup(func() { _ = session.Close() }) - valid := `{"version":8,"id":"request_local_edge","type":"prepare-local-edge","shareId":"share_123456","connectionId":"edge_1234567"}` + valid := `{"version":9,"id":"request_local_edge","type":"prepare-local-edge","shareId":"share_123456","connectionId":"edge_1234567"}` if _, err := session.Handle(t.Context(), []byte(valid)); err == nil || err.Error() != "native media source does not exist" { t.Fatalf("valid local-edge request stopped at wrong boundary: %v", err) } - invalid := `{"version":8,"id":"request_local_edge","type":"prepare-local-edge","shareId":"share_123456","connectionId":"edge_1234567","iceServers":[]}` + invalid := `{"version":9,"id":"request_local_edge","type":"prepare-local-edge","shareId":"share_123456","connectionId":"edge_1234567","iceServers":[]}` if _, err := session.Handle(t.Context(), []byte(invalid)); err == nil || err.Error() != "native prepare-local-edge request is invalid" { t.Fatalf("extended local-edge request was accepted: %v", err) } @@ -70,7 +314,7 @@ func TestPreviewFailureReturnsAnAdvisoryResponse(t *testing.T) { session := New("missing-capture-process", nativecapture.Capabilities{}, false) t.Cleanup(func() { _ = session.Close() }) value, err := session.Handle(t.Context(), []byte( - `{"version":8,"id":"request_preview","type":"source-preview","source":{"kind":"display","sourceId":"65537","title":"Display 1"}}`, + `{"version":9,"id":"request_preview","type":"source-preview","source":{"kind":"display","sourceId":"65537","title":"Display 1"}}`, )) if err != nil { t.Fatal(err) @@ -81,6 +325,24 @@ func TestPreviewFailureReturnsAnAdvisoryResponse(t *testing.T) { } } +func TestSourceEnumerationFailureDoesNotCloseControl(t *testing.T) { + session := New("missing-capture-process", nativecapture.Capabilities{}, false) + t.Cleanup(func() { _ = session.Close() }) + value, err := session.Handle(t.Context(), []byte( + `{"version":9,"id":"request_sources","type":"list-sources"}`, + )) + if err != nil || value != operationFailure(requestEnvelope{ + Version: loopback.ProtocolVersion, ID: "request_sources", Type: "list-sources", + }) { + t.Fatalf("source list failure ended its control session: %#v, %v", value, err) + } + if _, err := session.Handle(t.Context(), []byte( + `{"version":9,"id":"request_options","type":"capture-options"}`, + )); err != nil { + t.Fatalf("control was unusable after source enumeration failed: %v", err) + } +} + func TestQualitySettingsMapOnceIntoNativeMedia(t *testing.T) { settings := qualitySettings{ Resolution: "1440p", MaxFramerate: 60, MaxBitrate: 12_000_000, @@ -104,12 +366,12 @@ func TestQualitySettingsMapOnceIntoNativeMedia(t *testing.T) { func TestUpdateShareRequiresTheCurrentStrictProfile(t *testing.T) { session := New("missing-capture-process", nativecapture.Capabilities{}, false) t.Cleanup(func() { _ = session.Close() }) - valid := `{"version":8,"id":"request_update","type":"update-share","shareId":"share_123456","profile":{"resolution":"1080p","maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced","screenAudioQuality":"music"}}` + valid := `{"version":9,"id":"request_update","type":"update-share","shareId":"share_123456","profile":{"resolution":"1080p","maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced","screenAudioQuality":"music"}}` if _, err := session.Handle(t.Context(), []byte(valid)); err == nil || err.Error() != "native share does not exist" { t.Fatalf("valid update stopped at wrong boundary: %v", err) } - invalid := `{"version":8,"id":"request_update","type":"update-share","shareId":"share_123456","profile":{"resolution":"1080p","maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced","screenAudioQuality":"music","extra":true}}` + invalid := `{"version":9,"id":"request_update","type":"update-share","shareId":"share_123456","profile":{"resolution":"1080p","maxFramerate":30,"maxBitrate":5000000,"degradationPreference":"balanced","screenAudioQuality":"music","extra":true}}` if _, err := session.Handle(t.Context(), []byte(invalid)); err == nil || err.Error() != "native update-share request is invalid" { t.Fatalf("extended update was accepted: %v", err) @@ -119,14 +381,19 @@ func TestUpdateShareRequiresTheCurrentStrictProfile(t *testing.T) { func TestReplaceShareSourceKeepsOneStrictTargetShape(t *testing.T) { session := New("missing-capture-process", nativecapture.Capabilities{}, false) t.Cleanup(func() { _ = session.Close() }) - valid := `{"version":8,"id":"request_source","type":"replace-share-source","shareId":"share_123456","source":{"kind":"picker","sourceId":"1","title":"Portal"},"audio":false,"adapterIndex":0,"encoderIndex":0}` + valid := `{"version":9,"id":"request_source","type":"replace-share-source","shareId":"share_123456","source":{"kind":"picker","sourceId":"1","title":"Portal"},"audio":false,"adapterIndex":0,"encoderIndex":0}` if _, err := session.Handle(t.Context(), []byte(valid)); err == nil || err.Error() != "native share does not exist" { t.Fatalf("valid source replacement stopped at wrong boundary: %v", err) } - invalid := `{"version":8,"id":"request_source","type":"replace-share-source","shareId":"share_123456","source":{"kind":"picker","sourceId":"1","title":"Portal"},"audio":false,"adapterIndex":0,"encoderIndex":0,"extra":true}` + invalid := `{"version":9,"id":"request_source","type":"replace-share-source","shareId":"share_123456","source":{"kind":"picker","sourceId":"1","title":"Portal"},"audio":false,"adapterIndex":0,"encoderIndex":0,"extra":true}` if _, err := session.Handle(t.Context(), []byte(invalid)); err == nil || err.Error() != "native replace-share-source request is invalid" { t.Fatalf("extended source replacement was accepted: %v", err) } + invalid = strings.Replace(valid, `"kind":"picker"`, `"kind":"unknown"`, 1) + if _, err := session.Handle(t.Context(), []byte(invalid)); err == nil || + err.Error() != "native replace-share-source request is invalid" { + t.Fatalf("malformed target was treated as an operational failure: %v", err) + } } diff --git a/internal/client/nativecontrol/wire.go b/internal/client/nativecontrol/wire.go index 922e0da..657b8e1 100644 --- a/internal/client/nativecontrol/wire.go +++ b/internal/client/nativecontrol/wire.go @@ -1,6 +1,7 @@ package nativecontrol import ( + "github.com/TNTcraftHIM/Screener/internal/client/mediaedge" "github.com/TNTcraftHIM/Screener/internal/client/nativecapture" "github.com/pion/webrtc/v4" ) @@ -17,6 +18,11 @@ type responseEnvelope struct { Type string `json:"type"` } +type requestFailedResponse struct { + responseEnvelope + Code string `json:"code"` +} + type captureOptionsRequest requestEnvelope type listSourcesRequest requestEnvelope @@ -218,10 +224,52 @@ type receiveAnswerResponse struct { } type eventEnvelope struct { - Version int `json:"version"` - Type string `json:"type"` - ShareID string `json:"shareId"` - ConnectionID string `json:"connectionId,omitempty"` + Version int `json:"version"` + Type string `json:"type"` + ShareID string `json:"shareId"` + ConnectionID string `json:"connectionId,omitempty"` + PublicationGeneration string `json:"publicationGeneration,omitempty"` +} + +type publicationIdentity struct { + requestEnvelope + ShareID string `json:"shareId"` + PublicationGeneration string `json:"publicationGeneration"` + ConnectionID string `json:"connectionId"` +} + +type preparePublicationRequest struct { + publicationIdentity + ICEServers []iceServer `json:"iceServers"` +} + +type publicationAnswerRequest struct { + publicationIdentity + SDP string `json:"sdp"` +} + +type publicationCandidateRequest struct { + publicationIdentity + Candidate *webrtc.ICECandidateInit `json:"candidate"` +} + +type publicationLayersRequest struct { + publicationIdentity + ActiveCount *int `json:"activeCount"` +} + +type publicationResponse struct { + responseEnvelope + ShareID string `json:"shareId"` + PublicationGeneration string `json:"publicationGeneration"` + ConnectionID string `json:"connectionId"` + SDP string `json:"sdp,omitempty"` + Media mediaedge.PublicationMedia `json:"media"` +} + +type publicationQualityEvent struct { + eventEnvelope + mediaedge.PublicationQualitySample } type captureStateEvent struct { diff --git a/internal/client/nativehost/audio_test.go b/internal/client/nativehost/audio_test.go new file mode 100644 index 0000000..4cf12f3 --- /dev/null +++ b/internal/client/nativehost/audio_test.go @@ -0,0 +1,180 @@ +package nativehost + +import ( + "context" + "encoding/binary" + "encoding/json" + "io" + "os" + "sync/atomic" + "testing" + "time" + + "github.com/TNTcraftHIM/Screener/internal/client/nativeaudio" + "github.com/TNTcraftHIM/Screener/internal/client/nativecapture" + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/pion/webrtc/v4" +) + +func audioRecoveryProfile() nativecapture.VideoProfile { + return nativecapture.VideoProfile{Width: 1280, Height: 720, Framerate: 30, + Bitrate: 2_000_000, Preference: "balanced"} +} + +// The same stable test executable supplies framed video and PCM, without a +// capture device. Closing stdin ends either child independently. +func runAudioRecoveryCapture() { + stopped := make(chan struct{}) + go func() { _, _ = io.Copy(io.Discard, os.Stdin); close(stopped) }() + write := func(frame nativecapture.Frame) { + var header [32]byte + copy(header[:], "SMED") + header[4], header[5], header[7] = 2, byte(frame.Kind), byte(frame.Layer) + if frame.KeyFrame { + header[6] = 1 + } + binary.BigEndian.PutUint64(header[8:16], uint64(frame.Timestamp/(100*time.Nanosecond))) + binary.BigEndian.PutUint64(header[16:24], uint64(frame.Duration/(100*time.Nanosecond))) + binary.BigEndian.PutUint16(header[24:26], uint16(frame.Width)) + binary.BigEndian.PutUint16(header[26:28], uint16(frame.Height)) + binary.BigEndian.PutUint32(header[28:32], uint32(len(frame.Data))) + if _, err := os.Stdout.Write(append(header[:], frame.Data...)); err != nil { + os.Exit(1) + } + } + audio := len(os.Args) > 1 && os.Args[1] == "--capture-audio" + period := time.Second / 30 + status := []byte(`{"state":"active","audio":true}`) + if audio { + period = 20 * time.Millisecond + } else { + outputs := nativecapture.ScreenShareOutputs(audioRecoveryProfile()) + outputs = append(outputs, outputs[0]) + status, _ = json.Marshal(CaptureState{State: "active", Codec: "vp8", + Width: 1280, Height: 720, FPS: 30, Outputs: outputs}) + } + write(nativecapture.Frame{Kind: nativecapture.FrameStatus, Data: status}) + ticker := time.NewTicker(period) + defer ticker.Stop() + for pts := period; ; pts += period { + select { + case <-stopped: + return + case <-ticker.C: + } + if audio { + write(nativecapture.Frame{Kind: nativecapture.FramePCM, Timestamp: pts, + Duration: period, Data: make([]byte, nativeaudio.FrameBytes)}) + continue + } + write(nativecapture.Frame{Kind: nativecapture.FrameBegin, Timestamp: pts, Duration: period}) + for layer := range 2 { + write(nativecapture.Frame{Kind: nativecapture.FrameVP8, Layer: layer, Width: 8, Height: 8, + Timestamp: pts, Duration: period, KeyFrame: true, Data: sfu.VP8KeyFrame8x8}) + } + } +} + +func TestAudioEOFWaitsForExplicitSourceReplacement(t *testing.T) { + t.Setenv("SCREENER_NATIVEHOST_PIPE_FIXTURE", "audio-recovery") + check := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + ctx, cancel := context.WithTimeout(t.Context(), 15*time.Second) + defer cancel() + executable, err := os.Executable() + check(err) + events := make(chan Event, 64) + options := nativecapture.VideoOptions{Codec: "vp8", Profile: audioRecoveryProfile(), + Target: nativecapture.CaptureTarget{Kind: "display", SourceID: "1", Title: "Audio fixture"}} + session, err := Start(ctx, Options{ShareID: "audio-recovery", CaptureProcess: executable, + Video: options, Profile: QualityProfile{Video: options.Profile, AudioBitrate: 64_000}, + AudioEnabled: true, EdgeCapacity: 1, BindAddress: "127.0.0.1:0", Events: events}) + check(err) + defer session.Close() + if !session.HasAudio() { + t.Fatal("fixture did not start audio") + } + settings := webrtc.SettingEngine{} + settings.SetIncludeLoopbackCandidate(true) + settings.SetNetworkTypes([]webrtc.NetworkType{webrtc.NetworkTypeUDP4}) + receiver, err := webrtc.NewAPI(webrtc.WithSettingEngine(settings)).NewPeerConnection(webrtc.Configuration{}) + check(err) + defer receiver.Close() + var audioPackets, videoPackets atomic.Uint64 + receiver.OnTrack(func(track *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { + for { + packet, _, err := track.ReadRTP() + if err != nil { + return + } + if track.Codec().MimeType == webrtc.MimeTypeOpus && len(packet.Payload) > 0 { + audioPackets.Add(1) + } else if track.Kind() == webrtc.RTPCodecTypeVideo { + videoPackets.Add(1) + } + } + }) + offer, err := session.PrepareLocalEdge("audio-edge") + check(err) + check(receiver.SetRemoteDescription(offer)) + go func() { + for { + select { + case event := <-events: + if event.Type == "edge-candidate" && event.Candidate != nil { + _ = receiver.AddICECandidate(*event.Candidate) + } + case <-ctx.Done(): + return + } + } + }() + gathered := webrtc.GatheringCompletePromise(receiver) + answer, err := receiver.CreateAnswer(nil) + check(err) + check(receiver.SetLocalDescription(answer)) + select { + case <-gathered: + case <-ctx.Done(): + t.Fatal("receiver gathering timed out") + } + check(session.SetAnswer("audio-edge", *receiver.LocalDescription())) + waitPackets := func(counter *atomic.Uint64, minimum uint64, label string) { + t.Helper() + deadline := time.Now().Add(3 * time.Second) + for counter.Load() < minimum { + if time.Now().After(deadline) { + t.Fatalf("%s: received %d, wanted %d", label, counter.Load(), minimum) + } + time.Sleep(10 * time.Millisecond) + } + } + waitPackets(&audioPackets, 3, "initial Opus") + waitPackets(&videoPackets, 3, "initial video") + previousAudio, previousVideo := session.currentAudioStream(), session.currentStream() + edge, audioSource := session.edge("audio-edge"), session.audioSource + check(previousAudio.Close()) + waitPackets(&videoPackets, videoPackets.Load()+10, "video after audio EOF") + before := audioPackets.Load() + options.Target.SourceID = "2" + check(session.ReplaceSource(ctx, options, true)) + if session.currentStream() == previousVideo || session.currentAudioStream() == previousAudio || + session.edge("audio-edge") != edge || session.audioSource != audioSource { + t.Fatal("explicit replacement did not preserve the media owners") + } + waitPackets(&audioPackets, before+3, "Opus after explicit source replacement") + check(session.currentAudioStream().Close()) + waitPackets(&videoPackets, videoPackets.Load()+5, "video after second audio EOF") + closed := make(chan error, 1) + go func() { closed <- session.Close() }() + select { + case err = <-closed: + check(err) + case <-time.After(3 * time.Second): + t.Fatal("Stop did not unblock the audio replacement wait") + } +} diff --git a/internal/client/nativehost/publication.go b/internal/client/nativehost/publication.go new file mode 100644 index 0000000..381a3e3 --- /dev/null +++ b/internal/client/nativehost/publication.go @@ -0,0 +1,79 @@ +package nativehost + +import ( + "errors" + + "github.com/TNTcraftHIM/Screener/internal/client/mediaedge" + "github.com/pion/webrtc/v4" +) + +type publicationKey struct { + generation, connectionID string +} + +func (session *Session) PreparePublication(generation, connectionID string, servers []webrtc.ICEServer) (webrtc.SessionDescription, mediaedge.PublicationMedia, error) { + session.mu.Lock() + key := publicationKey{generation: generation, connectionID: connectionID} + if session.closed || len(session.publications) >= 2 || session.publications[key] != nil || session.source == nil { + session.mu.Unlock() + return webrtc.SessionDescription{}, mediaedge.PublicationMedia{}, errors.New("native publication is unavailable") + } + source, audio := session.source, session.audioSource + session.mu.Unlock() + publication, err := session.engine.NewPublication(source, mediaedge.EdgeOptions{ + ConnectionID: connectionID, ICEServers: servers, Audio: audio, + Events: mediaedge.EdgeEvents{ + LocalCandidate: func(candidate *webrtc.ICECandidateInit) { + session.emit(Event{Type: "publication-candidate", ShareID: session.shareID, + PublicationGeneration: generation, ConnectionID: connectionID, Candidate: candidate}) + }, + ConnectionState: func(state webrtc.PeerConnectionState, _ *mediaedge.SelectedPair) { + session.emit(Event{Type: "publication-state", ShareID: session.shareID, + PublicationGeneration: generation, ConnectionID: connectionID, State: state.String()}) + }, + }, + }) + if err != nil { + return webrtc.SessionDescription{}, mediaedge.PublicationMedia{}, err + } + session.mu.Lock() + if session.closed || len(session.publications) >= 2 || session.publications[key] != nil { + session.mu.Unlock() + _ = publication.Close() + return webrtc.SessionDescription{}, mediaedge.PublicationMedia{}, errors.New("native publication changed") + } + if session.publications == nil { + session.publications = make(map[publicationKey]*mediaedge.Publication) + } + session.publications[key] = publication + session.mu.Unlock() + offer, err := publication.CreateOffer() + if err != nil { + session.ClosePublication(generation, connectionID) + return webrtc.SessionDescription{}, mediaedge.PublicationMedia{}, err + } + return offer, publication.Media(), nil +} + +func (session *Session) Publication(generation, connectionID string) (*mediaedge.Publication, error) { + session.mu.Lock() + defer session.mu.Unlock() + publication := session.publications[publicationKey{generation: generation, connectionID: connectionID}] + if session.closed || publication == nil { + return nil, errors.New("native publication identity changed") + } + return publication, nil +} + +func (session *Session) ClosePublication(generation, connectionID string) { + session.mu.Lock() + key := publicationKey{generation: generation, connectionID: connectionID} + publication := session.publications[key] + if publication == nil { + session.mu.Unlock() + return + } + delete(session.publications, key) + session.mu.Unlock() + _ = publication.Close() +} diff --git a/internal/client/nativehost/publication_test.go b/internal/client/nativehost/publication_test.go new file mode 100644 index 0000000..894c210 --- /dev/null +++ b/internal/client/nativehost/publication_test.go @@ -0,0 +1,52 @@ +package nativehost + +import ( + "testing" + + "github.com/TNTcraftHIM/Screener/internal/client/mediaedge" +) + +func TestPublicationGenerationFencesNativeControl(t *testing.T) { + engine, err := mediaedge.NewEngine(mediaedge.EngineOptions{BindAddress: "127.0.0.1:0", IncludeLoopback: true}) + if err != nil { + t.Fatal(err) + } + defer engine.Close() + source, err := engine.NewSource("vp8", 3, 1, nil) + if err != nil { + t.Fatal(err) + } + defer source.Close() + if err = source.SetFormat(0, 320, 180); err != nil { + t.Fatal(err) + } + if err = source.ConfigureOutputs([]uint32{300_000}); err != nil { + t.Fatal(err) + } + session := &Session{shareID: "share_12345678", engine: engine, source: source, ctx: t.Context(), events: make(chan Event, 64)} + _, media, err := session.PreparePublication("generation_12345678", "connection_12345678", nil) + if err != nil || media.Codec != "vp8" || len(media.Layers) != 1 { + t.Fatalf("publication did not reuse capture metadata: %+v %v", media, err) + } + if _, err = session.Publication("stale_generation", "connection_12345678"); err == nil { + t.Fatal("stale generation accessed publication") + } + session.ClosePublication("stale_generation", "connection_12345678") + if _, err = session.Publication("generation_12345678", "connection_12345678"); err != nil { + t.Fatal("stale close retired current publication") + } + if _, _, err = session.PreparePublication("candidate_generation", "candidate_connection", nil); err != nil { + t.Fatal("candidate preparation displaced the current publication", err) + } + if _, _, err = session.PreparePublication("third_generation", "third_connection", nil); err == nil { + t.Fatal("third publication was admitted") + } + session.ClosePublication("candidate_generation", "candidate_connection") + if _, err = session.Publication("generation_12345678", "connection_12345678"); err != nil { + t.Fatal("candidate rollback retired current publication") + } + session.ClosePublication("generation_12345678", "connection_12345678") + if _, err = session.Publication("generation_12345678", "connection_12345678"); err == nil { + t.Fatal("closed publication remained accessible") + } +} diff --git a/internal/client/nativehost/session.go b/internal/client/nativehost/session.go index 621fa9c..b567f4f 100644 --- a/internal/client/nativehost/session.go +++ b/internal/client/nativehost/session.go @@ -5,7 +5,10 @@ import ( "context" "encoding/json" "errors" + "fmt" "io" + "log/slog" + "slices" "strings" "sync" "time" @@ -13,6 +16,7 @@ import ( "github.com/TNTcraftHIM/Screener/internal/client/mediaedge" "github.com/TNTcraftHIM/Screener/internal/client/nativecapture" + "github.com/TNTcraftHIM/Screener/internal/media/encoded" "github.com/pion/webrtc/v4" ) @@ -20,32 +24,35 @@ const startTimeout = 5 * time.Second const qualitySamplePeriod = 2 * time.Second type CaptureState struct { - State string `json:"state"` - HardwareOnly bool `json:"hardwareOnly"` - Codec string `json:"codec"` - AdapterIndex *uint32 `json:"adapterIndex,omitempty"` - AdapterName string `json:"adapterName,omitempty"` - AdapterIdentity string `json:"adapterIdentity,omitempty"` - EncoderIndex *uint32 `json:"encoderIndex,omitempty"` - EncoderName string `json:"encoderName,omitempty"` - EncoderIdentity string `json:"encoderIdentity,omitempty"` - ProfileLevelID string `json:"profileLevelId,omitempty"` - Width uint32 `json:"width,omitempty"` - Height uint32 `json:"height,omitempty"` - FPS uint32 `json:"fps,omitempty"` - RestoreToken string `json:"restoreToken,omitempty"` + State string `json:"state"` + HardwareOnly bool `json:"hardwareOnly"` + Codec string `json:"codec"` + AdapterIndex *uint32 `json:"adapterIndex,omitempty"` + AdapterName string `json:"adapterName,omitempty"` + AdapterIdentity string `json:"adapterIdentity,omitempty"` + EncoderIndex *uint32 `json:"encoderIndex,omitempty"` + EncoderName string `json:"encoderName,omitempty"` + EncoderIdentity string `json:"encoderIdentity,omitempty"` + ProfileLevelID string `json:"profileLevelId,omitempty"` + Width uint32 `json:"width,omitempty"` + Height uint32 `json:"height,omitempty"` + FPS uint32 `json:"fps,omitempty"` + RestoreToken string `json:"restoreToken,omitempty"` + Outputs []nativecapture.OutputProfile `json:"outputs"` } type Event struct { - Type string - ShareID string - ConnectionID string - Candidate *webrtc.ICECandidateInit - State string - LocalType string - RemoteType string - NatTraversalPath string - Quality *mediaedge.QualitySample + Type string + ShareID string + ConnectionID string + PublicationGeneration string + Candidate *webrtc.ICECandidateInit + State string + LocalType string + RemoteType string + NatTraversalPath string + Quality *mediaedge.QualitySample + PublicationQuality *mediaedge.PublicationQualitySample } type Options struct { @@ -78,11 +85,14 @@ type Session struct { done chan error ready chan error - mu sync.Mutex - updateMu sync.Mutex - edges map[string]*mediaedge.Edge - paused bool - closed bool + mu sync.Mutex + updateMu sync.Mutex + edges map[string]*mediaedge.Edge + publications map[publicationKey]*mediaedge.Publication + captureApplied chan struct{} + audioChanged chan struct{} + paused bool + closed bool } func Start(parent context.Context, options Options) (*Session, error) { @@ -104,6 +114,7 @@ func Start(parent context.Context, options Options) (*Session, error) { if err != nil { return nil, err } + options.Video.OutputGroups = options.EdgeCapacity stream, err := nativecapture.StartVideo(parent, options.CaptureProcess, options.Video) if err != nil { _ = engine.Close() @@ -133,6 +144,7 @@ func Start(parent context.Context, options Options) (*Session, error) { done: make(chan error, 1), ready: make(chan error, 1), edges: make(map[string]*mediaedge.Edge), + audioChanged: make(chan struct{}, 1), } if audioStream != nil { audioSource, audioErr := engine.NewAudioSource( @@ -305,7 +317,7 @@ func (session *Session) UpdateProfile(profile QualityProfile) error { defer session.updateMu.Unlock() session.mu.Lock() - if session.closed { + if session.closed || session.ctx.Err() != nil { session.mu.Unlock() return errors.New("native share is unavailable") } @@ -324,7 +336,7 @@ func (session *Session) UpdateProfile(profile QualityProfile) error { } } session.mu.Lock() - if session.closed { + if session.closed || session.ctx.Err() != nil { session.mu.Unlock() return errors.New("native share is unavailable") } @@ -333,7 +345,8 @@ func (session *Session) UpdateProfile(profile QualityProfile) error { return nil } - replacement, state, err := session.prepareVideo(session.ctx, options) + // Updating an existing source never waits for an interactive picker. + replacement, state, err := session.prepareVideo(session.ctx, options, false, true) if err != nil { return err } @@ -369,6 +382,7 @@ func (session *Session) ReplaceSource( } profile := session.profile options.Codec = session.videoOptions.Codec + options.OutputGroups = session.edgeCapacity hasAudio := session.audioSource != nil && session.audioStream != nil session.mu.Unlock() if audioEnabled != hasAudio { @@ -376,7 +390,7 @@ func (session *Session) ReplaceSource( } options.Profile = profile.Video options.RestoreToken = "" - replacement, state, err := session.prepareVideo(ctx, options) + replacement, state, err := session.prepareVideo(ctx, options, options.Target.Kind == "picker", false) if err != nil { return err } @@ -416,21 +430,26 @@ func startAudioCapture( func (session *Session) prepareVideo( ctx context.Context, options nativecapture.VideoOptions, + interactive bool, + allowQuiet bool, ) (*nativecapture.Stream, CaptureState, error) { + options.OutputGroups = session.edgeCapacity replacement, err := nativecapture.StartVideo( ctx, session.captureProcess, options, ) if err != nil { - return nil, CaptureState{}, errors.New("native capture could not start") + return nil, CaptureState{}, captureProfileFailure(ctx, options.Profile, "process-start", + errors.New("native capture could not start")) } state, err := waitForCaptureProfile( ctx, replacement, options.Profile, options.Codec, - options.Target.Kind == "picker", + interactive, + allowQuiet, ) if err != nil { _ = replacement.Close() @@ -448,7 +467,7 @@ func (session *Session) commitCapture( replaceAudio bool, ) error { session.mu.Lock() - if session.closed { + if session.closed || session.ctx.Err() != nil { session.mu.Unlock() _ = replacement.Close() if replacementAudio != nil { @@ -462,22 +481,51 @@ func (session *Session) commitCapture( options.RestoreToken = state.RestoreToken } session.stream = replacement + applied := make(chan struct{}) + session.captureApplied = applied if replaceAudio { session.audioStream = replacementAudio + select { + case session.audioChanged <- struct{}{}: + default: + } } session.videoOptions = options session.profile = profile - session.source.SetFormat(state.Width, state.Height) session.mu.Unlock() - session.emit(Event{ - Type: "capture-state", ShareID: session.shareID, State: "active", - }) - _ = replacement.RequestKeyFrame() + _ = replacement.RequestKeyFrame(-1) _ = previous.Close() if replaceAudio && previousAudio != nil { _ = previousAudio.Close() } + select { + case <-applied: + case <-session.ctx.Done(): + return session.ctx.Err() + } + if err := session.ctx.Err(); err != nil { + return err + } + session.emit(Event{ + Type: "capture-state", ShareID: session.shareID, State: "active", + }) + return nil +} + +// Only the video reader installs source metadata, after leaving the old input. +// The update response waits for this exact stream's installation, not its process. +func (session *Session) installCapture(next *nativecapture.Stream) error { + session.source.BeginGeneration() + if err := configureCaptureOutputs(session.source, next.Outputs()); err != nil { + return err + } + session.mu.Lock() + if session.stream == next && session.captureApplied != nil { + close(session.captureApplied) + session.captureApplied = nil + } + session.mu.Unlock() return nil } @@ -496,6 +544,7 @@ func (session *Session) Done() <-chan error { } func (session *Session) Close() error { + session.cancel() session.updateMu.Lock() defer session.updateMu.Unlock() session.mu.Lock() @@ -505,25 +554,18 @@ func (session *Session) Close() error { return nil } session.closed = true - edges := make([]*mediaedge.Edge, 0, len(session.edges)) - for _, edge := range session.edges { - edges = append(edges, edge) - } + source := session.source stream := session.stream audioStream := session.audioStream session.edges = make(map[string]*mediaedge.Edge) session.mu.Unlock() - session.cancel() - for _, edge := range edges { - _ = edge.Close() + if source != nil { + _ = source.Close() } _ = stream.Close() if audioStream != nil { _ = audioStream.Close() } - if session.source != nil { - _ = session.source.Close() - } if session.audioSource != nil { _ = session.audioSource.Close() } @@ -564,9 +606,9 @@ func (session *Session) run() { session.mu.Lock() closed := session.closed session.mu.Unlock() - if closed { - // An explicit Close is a clean end, even if closing the capture process - // made the video reader return an error. + if closed || session.ctx.Err() != nil { + // Closing this owner or its parent is a clean end, even if stopping the + // capture process made the video reader return an error. result = nil } session.cancel() @@ -594,6 +636,9 @@ func (session *Session) run() { if audioStream != nil { _ = audioStream.Close() } + if slog.Default().Enabled(session.ctx, slog.LevelDebug) { + slog.Debug("screener-client", "event", "share-ended", "failed", result != nil, "errorType", fmt.Sprintf("%T", result)) + } session.done <- result close(session.done) }() @@ -602,6 +647,7 @@ func (session *Session) run() { func (session *Session) runVideo() error { ready := false + activeSeen := false fail := func(err error) error { if !ready { session.ready <- err @@ -609,21 +655,32 @@ func (session *Session) runVideo() error { return err } current := session.currentStream() + changeStream := func(next *nativecapture.Stream) error { + if err := session.installCapture(next); err != nil { + return err + } + current = next + // prepareVideo consumed and verified this replacement's active status. + activeSeen = true + return nil + } for current != nil { frame, err := current.Read() if err != nil { next := session.currentStream() if next != nil && next != current { - session.source.BeginGeneration() - current = next + if err = changeStream(next); err != nil { + return fail(err) + } continue } return fail(errors.New("native capture process stopped unexpectedly")) } if next := session.currentStream(); next != current { if next != nil { - session.source.BeginGeneration() - current = next + if err = changeStream(next); err != nil { + return fail(err) + } } continue } @@ -633,70 +690,102 @@ func (session *Session) runVideo() error { if statusErr != nil { return fail(statusErr) } + if !slices.Equal(status.Outputs, current.Outputs()) { + return fail(errors.New("native capture outputs were not applied")) + } session.mu.Lock() if session.closed { session.mu.Unlock() return fail(errors.New("native share stopped")) } + if session.stream != current { + session.mu.Unlock() + continue + } if session.source == nil { if session.videoOptions.Codec != "auto" && session.videoOptions.Codec != status.Codec { session.mu.Unlock() return fail(errors.New("native capture codec was not applied")) } - source, sourceErr := session.engine.NewSource(status.Codec, session.edgeCapacity, func() { + source, sourceErr := session.engine.NewSource(status.Codec, session.edgeCapacity, min(2, len(status.Outputs)), func() { if stream := session.currentStream(); stream != nil { - _ = stream.RequestKeyFrame() + _ = stream.RequestKeyFrame(-1) } }) if sourceErr != nil { session.mu.Unlock() return fail(sourceErr) } + if sourceErr = configureCaptureOutputs(source, status.Outputs); sourceErr != nil { + session.mu.Unlock() + _ = source.Close() + return fail(sourceErr) + } session.source = source session.videoOptions.Codec = status.Codec } else if session.source.Codec() != status.Codec { session.mu.Unlock() return fail(errors.New("native capture codec changed within a share")) } - session.mu.Unlock() - session.emit(Event{Type: "capture-state", ShareID: session.shareID, State: status.State}) if status.State == "active" { - session.mu.Lock() - profile := session.videoOptions.Profile + activeSeen = true if status.RestoreToken != "" { session.videoOptions.RestoreToken = status.RestoreToken } - session.mu.Unlock() - if status.Width != profile.Width || status.Height != profile.Height || - status.FPS != profile.Framerate { - return fail(errors.New("native capture profile was not applied")) - } - session.source.SetFormat(status.Width, status.Height) } + session.mu.Unlock() + slog.Debug("screener-client", "event", "capture-state", "state", status.State, "codec", status.Codec, + "width", status.Width, "height", status.Height, "fps", status.FPS) + session.emit(Event{Type: "capture-state", ShareID: session.shareID, State: status.State}) if !ready { ready = true session.ready <- nil } + case nativecapture.FrameBegin: + if session.source == nil { + return fail(errors.New("native capture input arrived before its state")) + } + plan, beginErr := session.source.BeginFrame(frame.Timestamp) + if beginErr != nil { + return fail(beginErr) + } + session.mu.Lock() + paused := session.paused + session.mu.Unlock() + if paused { + clear(plan.Active) + } + controlErr := applyOutputPlan(current, plan, activeSeen) + if controlErr != nil && session.currentStream() == current { + return fail(controlErr) + } case nativecapture.FrameH264, nativecapture.FrameVP8: if session.source == nil || (frame.Kind == nativecapture.FrameH264) != (session.source.Codec() == "h264") { return fail(errors.New("native video frame does not match its source codec")) } - // Pion writes every binding before returning a per-binding error. Edge - // state owns that failure; one retired edge must not stop healthy siblings. + if formatErr := session.source.SetFormat(frame.Layer, frame.Width, frame.Height); formatErr != nil { + return fail(formatErr) + } + // Edge state owns write failures; one retired edge cannot stop siblings. session.mu.Lock() paused := session.paused session.mu.Unlock() if !paused { - writeErr := session.source.WriteVideo( - frame.Data, - frame.Timestamp, - frame.Duration, - ) - if errors.Is(writeErr, mediaedge.ErrInvalidVideoTimestamp) { + writeErr := session.source.WriteVideo(frame.Layer, encoded.Frame{ + Data: frame.Data, PTS: frame.Timestamp, Duration: frame.Duration, Recovery: frame.KeyFrame, + }) + if errors.Is(writeErr, encoded.ErrInvalidTimestamp) { return fail(writeErr) } } + case nativecapture.FrameLayerUnavailable: + if session.source == nil { + return fail(errors.New("native output ended before its source state")) + } + if err = session.source.DisableLayer(frame.Layer); err != nil { + return fail(err) + } case nativecapture.FramePCM: return fail(errors.New("native video process emitted audio")) default: @@ -706,6 +795,53 @@ func (session *Session) runVideo() error { return fail(errors.New("native capture process stopped unexpectedly")) } +func configureCaptureOutputs(source *mediaedge.Source, outputs []nativecapture.OutputProfile) error { + ceilings := make([]uint32, len(outputs)) + for layer, output := range outputs { + if layer < 2 { + if err := source.SetFormat(layer, output.Width, output.Height); err != nil { + return err + } + } + ceilings[layer] = output.Bitrate + } + if err := source.ConfigureOutputs(ceilings); err != nil { + return err + } + for layer := 2; layer < len(outputs); layer++ { + if err := source.SetFormat(layer, outputs[layer].Width, outputs[layer].Height); err != nil { + return err + } + } + source.SetLowestLayerRateControlled(true) + return nil +} + +func applyOutputPlan(stream *nativecapture.Stream, plan mediaedge.OutputPlan, activeSeen bool) error { + for layer, bitrate := range plan.Bitrates { + if bitrate > 0 { + if err := stream.SetOutputBitrate(layer, bitrate); err != nil { + return err + } + } + } + for _, layer := range plan.RecoveryLayers { + if err := stream.RequestKeyFrame(layer); err != nil { + return err + } + } + // A new group must receive its budget before activation admits a frame. + // Startup retains the original until its first active status has arrived. + if activeSeen { + for layer, active := range plan.Active { + if err := stream.SetOutputActive(layer, active); err != nil { + return err + } + } + } + return nil +} + func (session *Session) currentStream() *nativecapture.Stream { session.mu.Lock() defer session.mu.Unlock() @@ -715,40 +851,81 @@ func (session *Session) currentStream() *nativecapture.Stream { return session.stream } +func captureProfileFailure(ctx context.Context, profile nativecapture.VideoProfile, stage string, err error) error { + slog.DebugContext(ctx, "screener-client", "event", "capture-profile-rejected", "stage", stage, + "width", profile.Width, "height", profile.Height, "fps", profile.Framerate, "bitrate", profile.Bitrate) + return err +} + func waitForCaptureProfile( ctx context.Context, stream *nativecapture.Stream, profile nativecapture.VideoProfile, codec string, interactive bool, + allowQuiet bool, ) (CaptureState, error) { type result struct { state CaptureState err error + stage string + input bool } ready := make(chan result, 1) + stopped := make(chan struct{}) + defer close(stopped) go func() { + send := func(value result) bool { + select { + case ready <- value: + return true + case <-stopped: + return false + } + } + inputSeen := false for { frame, err := stream.Read() if err != nil { - ready <- result{err: errors.New("native capture profile did not become ready")} + send(result{stage: "process-ended", err: errors.New("native capture profile did not become ready")}) return } + if frame.Kind == nativecapture.FrameLayerUnavailable { + send(result{stage: "output-unavailable", err: errors.New("native capture profile has an unavailable output")}) + return + } + if frame.Kind == nativecapture.FrameBegin && !inputSeen { + inputSeen = true + if !send(result{input: true}) { + return + } + } if frame.Kind != nativecapture.FrameStatus { continue } state, err := decodeCaptureState(frame.Data) if err != nil { - ready <- result{err: err} + send(result{stage: "status-invalid", err: err}) + return + } + if !slices.Equal(state.Outputs, stream.Outputs()) { + send(result{stage: "outputs-mismatch", err: errors.New("native capture outputs were not applied")}) + return + } + if state.Codec != codec { + send(result{stage: "profile-mismatch", err: errors.New("native capture codec was not applied")}) return } if state.State == "active" { - if state.Codec != codec || state.Width != profile.Width || state.Height != profile.Height || + if state.Width != profile.Width || state.Height != profile.Height || state.FPS != profile.Framerate { - ready <- result{err: errors.New("native capture profile was not applied")} + send(result{stage: "profile-mismatch", err: errors.New("native capture profile was not applied")}) return } - ready <- result{state: state} + send(result{state: state}) + return + } + if !send(result{state: state}) { return } } @@ -760,13 +937,38 @@ func waitForCaptureProfile( timeout = timer.C defer timer.Stop() } - select { - case value := <-ready: - return value.state, value.err - case <-timeout: - return CaptureState{}, errors.New("native capture profile timed out") - case <-ctx.Done(): - return CaptureState{}, errors.New("native share stopped") + inputSeen, startingSeen := false, false + for { + select { + case value := <-ready: + if value.err != nil { + return value.state, captureProfileFailure(ctx, profile, value.stage, value.err) + } + if value.state.State == "active" { + return value.state, nil + } + if allowQuiet && timer != nil { + if value.input && !inputSeen { + if !startingSeen { + return CaptureState{}, captureProfileFailure(ctx, profile, "status-invalid", + errors.New("native capture input arrived before its starting state")) + } + inputSeen = true + timer.Reset(startTimeout) + timeout = timer.C + } else if value.state.State == "starting" && !inputSeen { + startingSeen = true + timer.Stop() + timeout = nil + } + } + case <-timeout: + return CaptureState{}, captureProfileFailure(ctx, profile, "wait-timeout", + errors.New("native capture profile timed out")) + case <-ctx.Done(): + return CaptureState{}, captureProfileFailure(ctx, profile, "share-stopped", + errors.New("native share stopped")) + } } } @@ -781,6 +983,10 @@ func (session *Session) runQuality() { for _, edge := range session.edges { edges = append(edges, edge) } + publications := make(map[publicationKey]*mediaedge.Publication, len(session.publications)) + for key, publication := range session.publications { + publications[key] = publication + } session.mu.Unlock() for _, edge := range edges { if sample, ok := edge.QualitySample(now); ok { @@ -790,6 +996,12 @@ func (session *Session) runQuality() { }) } } + for key, publication := range publications { + if sample, ok := publication.QualitySample(now); ok { + session.emit(Event{Type: "publication-quality", ShareID: session.shareID, + PublicationGeneration: key.generation, ConnectionID: key.connectionID, PublicationQuality: &sample}) + } + } case <-session.ctx.Done(): return } @@ -800,30 +1012,28 @@ func (session *Session) runAudio() { current := session.currentAudioStream() for current != nil { frame, err := current.Read() - if err != nil { - next := session.currentAudioStream() - if next != nil && next != current { - current = next - continue - } - return - } if next := session.currentAudioStream(); next != current { current = next continue } - if frame.Kind != nativecapture.FramePCM { - return + if err == nil && frame.Kind == nativecapture.FramePCM { + session.mu.Lock() + paused := session.paused + session.mu.Unlock() + if paused || session.audioSource.WritePCM(frame.Data, frame.Duration) == nil { + continue + } } - session.mu.Lock() - paused := session.paused - session.mu.Unlock() - if paused { - continue - } - if err = session.audioSource.WritePCM(frame.Data, frame.Duration); err != nil { - return + // Audio failure leaves video live. Only an explicit source replacement + // changes this owner; its buffered wake also covers an earlier commit. + for session.currentAudioStream() == current { + select { + case <-session.audioChanged: + case <-session.ctx.Done(): + return + } } + current = session.currentAudioStream() } } @@ -853,19 +1063,27 @@ func decodeCaptureState(payload []byte) (CaptureState, error) { if err := decoder.Decode(&state); err != nil || decoder.Decode(&struct{}{}) != io.EOF || (state.State != "starting" && state.State != "active") || (state.Codec != "h264" && state.Codec != "vp8") || - state.HardwareOnly != (state.Codec == "h264") { + state.HardwareOnly != (state.Codec == "h264") || len(state.Outputs) < 1 || len(state.Outputs) > 6 { return CaptureState{}, errors.New("native capture state is invalid") } + for _, output := range state.Outputs { + if !output.Valid() { + return CaptureState{}, errors.New("native capture output profile is invalid") + } + } if state.State == "starting" && (state.AdapterIndex == nil || (state.Codec == "h264" && state.EncoderIndex == nil) || state.AdapterName == "" || state.AdapterIdentity == "" || state.EncoderName == "" || state.EncoderIdentity == "") { return CaptureState{}, errors.New("native capture starting state is incomplete") } + original := state.Outputs[min(1, len(state.Outputs)-1)] if state.State == "active" && ((state.Codec == "h264" && !validH264ProfileLevelID(state.ProfileLevelID)) || (state.Codec == "vp8" && state.ProfileLevelID != "") || - state.Width == 0 || state.Height == 0 || state.FPS == 0 || + state.Width != original.Width || + state.Height != original.Height || + state.FPS != original.Framerate || len(state.RestoreToken) > 4096 || !utf8.ValidString(state.RestoreToken) || strings.ContainsRune(state.RestoreToken, 0)) { return CaptureState{}, errors.New("native capture active state is incomplete") diff --git a/internal/client/nativehost/session_test.go b/internal/client/nativehost/session_test.go index 115d9cf..bd29a36 100644 --- a/internal/client/nativehost/session_test.go +++ b/internal/client/nativehost/session_test.go @@ -1,15 +1,193 @@ package nativehost -import "testing" +import ( + "bytes" + "context" + "encoding/json" + "errors" + "io" + "log/slog" + "os" + "strings" + "testing" + "time" + + "github.com/TNTcraftHIM/Screener/internal/client/mediaedge" + "github.com/TNTcraftHIM/Screener/internal/client/nativecapture" +) + +func TestMain(tests *testing.M) { + if os.Getenv("SCREENER_NATIVEHOST_PIPE_FIXTURE") == "audio-recovery" { + runAudioRecoveryCapture() + os.Exit(0) + } + if os.Getenv("SCREENER_NATIVEHOST_PIPE_FIXTURE") == "echo" { + _, _ = io.Copy(os.Stdout, os.Stdin) + os.Exit(0) + } + if os.Getenv("SCREENER_NATIVEHOST_PIPE_FIXTURE") == "1" { + _, _ = io.Copy(io.Discard, os.Stdin) + os.Exit(0) + } + os.Exit(tests.Run()) +} + +func TestCaptureCommitWaitsForReaderMetadataOrTermination(t *testing.T) { + t.Setenv("SCREENER_NATIVEHOST_PIPE_FIXTURE", "1") + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + for _, mode := range []string{"installed", "reader-failed", "closed"} { + t.Run(mode, func(t *testing.T) { + check := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + engine, err := mediaedge.NewEngine(mediaedge.EngineOptions{BindAddress: "127.0.0.1:0", IncludeLoopback: true}) + check(err) + defer engine.Close() + options := nativecapture.VideoOptions{Target: nativecapture.CaptureTarget{Kind: "display", SourceID: "1", Title: "Fixture"}, Codec: "vp8", + Profile: nativecapture.VideoProfile{Width: 1280, Height: 720, Framerate: 15, Bitrate: 2_000_000, Preference: "balanced"}, OutputGroups: 1} + previous, err := nativecapture.StartVideo(ctx, executable, options) + check(err) + defer previous.Close() + if len(previous.Outputs()) != 3 { + t.Fatal("capture did not retain its independent output group") + } + source, err := engine.NewSource("vp8", 1, 2, nil) + check(err) + defer source.Close() + check(configureCaptureOutputs(source, previous.Outputs())) + publication, err := engine.NewPublication(source, mediaedge.EdgeOptions{ConnectionID: "metadata-fixture"}) + check(err) + session := &Session{ctx: ctx, cancel: cancel, engine: engine, source: source, stream: previous, + edgeCapacity: 1, ready: make(chan error, 1), done: make(chan error, 1), events: make(chan Event, 16)} + options.Profile.Width, options.Profile.Height = 854, 480 + replacement, err := nativecapture.StartVideo(ctx, executable, options) + check(err) + defer replacement.Close() + committed := make(chan error, 1) + go func() { + session.updateMu.Lock() + err := session.commitCapture(options, QualityProfile{Video: options.Profile, AudioBitrate: 64_000}, replacement, CaptureState{}, nil, false) + session.updateMu.Unlock() + committed <- err + }() + select { + case <-previous.Done(): + case <-time.After(5 * time.Second): + t.Fatal("old fixture did not exit") + } + select { + case err := <-committed: + t.Fatalf("process exit acknowledged metadata before reader installation: %v", err) + default: + } + if len(publication.Media().Layers) != 2 || publication.Media().Layers[1].Width != 1280 { + t.Fatal("held reader unexpectedly changed source metadata") + } + if mode == "installed" { + // Hold the reader at its handoff, then run the same installation it uses. + check(session.installCapture(replacement)) + select { + case err = <-committed: + check(err) + case <-time.After(time.Second): + t.Fatal("installed metadata did not acknowledge update") + } + media := publication.Media() + if media.Layers[1].Width != 854 || media.Layers[1].Height != 480 || media.Layers[1].Bitrate != 2_000_000 { + t.Fatalf("acknowledged stale metadata: %+v", media) + } + return + } + go session.run() + closed := make(chan error, 1) + if mode == "closed" { + go func() { closed <- session.Close() }() + } else { + check(replacement.Close()) + } + select { + case err = <-committed: + if err == nil { + t.Fatal("terminated reader acknowledged an uninstalled stream") + } + case <-time.After(5 * time.Second): + t.Fatal("capture update remained latched after termination") + } + if mode == "closed" { + select { + case err = <-closed: + check(err) + case <-time.After(5 * time.Second): + t.Fatal("Close deadlocked behind the capture update") + } + } else { + select { + case err = <-session.Done(): + if err == nil { + t.Fatal("unexpected reader death did not fail the share") + } + case <-time.After(5 * time.Second): + t.Fatal("reader death did not end the share") + } + } + }) + } +} + +func TestOutputPlanPreservesOriginalStartupAndIndependentActivation(t *testing.T) { + t.Setenv("SCREENER_NATIVEHOST_PIPE_FIXTURE", "echo") + executable, err := os.Executable() + if err != nil { + t.Fatal(err) + } + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + stream, err := nativecapture.StartVideo(ctx, executable, nativecapture.VideoOptions{ + Target: nativecapture.CaptureTarget{Kind: "display", SourceID: "1", Title: "Fixture"}, Codec: "vp8", + Profile: nativecapture.VideoProfile{Width: 1280, Height: 720, Framerate: 15, + Bitrate: 2_000_000, Preference: "balanced"}, OutputGroups: 2, + }) + if err != nil { + t.Fatal(err) + } + defer stream.Close() + plan := mediaedge.OutputPlan{Active: []bool{false, true, true, false}, Bitrates: []uint32{0, 0, 250_000, 0}} + if err = applyOutputPlan(stream, plan, false); err != nil { + t.Fatal(err) + } + plan.RecoveryLayers = []int{2} + if err = applyOutputPlan(stream, plan, true); err != nil { + t.Fatal(err) + } + clear(plan.Active) + plan.RecoveryLayers = nil + if err = applyOutputPlan(stream, plan, true); err != nil { + t.Fatal(err) + } + for _, command := range []string{"B 2 250000", "K 2", "A 0 0", "A 2 1", "A 1 0", "A 2 0"} { + frame, readErr := stream.Read() + if readErr != nil || frame.Kind != nativecapture.FrameControl || string(frame.Data) != command { + t.Fatalf("expected control %q, received %+v, %v", command, frame, readErr) + } + } +} func TestCaptureStateKeepsStartingAndActiveContractsDistinct(t *testing.T) { - starting, err := decodeCaptureState([]byte( + starting, err := decodeCaptureState(captureStatePayload(t, `{"state":"starting","hardwareOnly":true,"codec":"h264","adapterIndex":0,"adapterName":"GPU","adapterIdentity":"0:1","encoderIndex":0,"encoderName":"H264","encoderIdentity":"encoder"}`, )) if err != nil || starting.State != "starting" || starting.AdapterIndex == nil { t.Fatalf("starting = %+v, %v", starting, err) } - active, err := decodeCaptureState([]byte( + active, err := decodeCaptureState(captureStatePayload(t, `{"state":"active","hardwareOnly":true,"codec":"h264","profileLevelId":"42c01f","width":1280,"height":720,"fps":30}`, )) if err != nil || active.State != "active" || active.ProfileLevelID != "42c01f" { @@ -21,7 +199,7 @@ func TestCaptureStateKeepsStartingAndActiveContractsDistinct(t *testing.T) { `{"state":"active","hardwareOnly":true,"codec":"h264","profileLevelId":"42c01e","width":854,"height":480,"fps":15}`, `{"state":"active","hardwareOnly":true,"codec":"h264","profileLevelId":"42c033","width":2560,"height":1440,"fps":60,"restoreToken":"portal-token"}`, } { - if _, err = decodeCaptureState([]byte(payload)); err != nil { + if _, err = decodeCaptureState(captureStatePayload(t, payload)); err != nil { t.Fatalf("valid profile state was rejected: %s: %v", payload, err) } } @@ -35,15 +213,87 @@ func TestCaptureStateRejectsUnknownOrUnattributedState(t *testing.T) { `{"state":"ready","hardwareOnly":true}`, `{"state":"active","hardwareOnly":true,"codec":"h264","extra":true}`, } { - if _, err := decodeCaptureState([]byte(payload)); err == nil { + if _, err := decodeCaptureState(captureStatePayload(t, payload)); err == nil { t.Fatalf("invalid state was accepted: %s", payload) } } } +func TestCaptureStateUsesOriginalSlotAndBoundsPhysicalGroups(t *testing.T) { + state, err := decodeCaptureState(captureStatePayload(t, + `{"state":"active","hardwareOnly":false,"codec":"vp8","width":1280,"height":720,"fps":30}`)) + if err != nil || len(state.Outputs) != 6 || state.Outputs[5] != state.Outputs[0] { + t.Fatalf("independent output groups = %+v, %v", state, err) + } + wrongOriginal := state + wrongOriginal.Width = state.Outputs[5].Width + tooMany := state + tooMany.Outputs = append(tooMany.Outputs, tooMany.Outputs[0]) + for _, invalid := range []CaptureState{wrongOriginal, tooMany} { + payload, err := json.Marshal(invalid) + if err != nil { + t.Fatal(err) + } + if _, err = decodeCaptureState(payload); err == nil { + t.Fatal("invalid original slot or output count was accepted") + } + } +} + +func captureStatePayload(t *testing.T, payload string) []byte { + t.Helper() + var fields map[string]json.RawMessage + if err := json.Unmarshal([]byte(payload), &fields); err != nil { + t.Fatal(err) + } + profile := nativecapture.VideoProfile{Width: 1280, Height: 720, Framerate: 30, Bitrate: 3_000_000} + for key, field := range map[string]*uint32{"width": &profile.Width, "height": &profile.Height, "fps": &profile.Framerate} { + if value, ok := fields[key]; ok { + if err := json.Unmarshal(value, field); err != nil { + t.Fatal(err) + } + } + } + slots := nativecapture.ScreenShareOutputs(profile) + for range 4 { + slots = append(slots, slots[0]) + } + outputs, err := json.Marshal(slots) + if err != nil { + t.Fatal(err) + } + fields["outputs"] = outputs + result, err := json.Marshal(fields) + if err != nil { + t.Fatal(err) + } + return result +} + func TestCandidateForRetiredEdgeIsIgnored(t *testing.T) { session := &Session{} if err := session.AddCandidate("retired-edge", nil); err != nil { t.Fatalf("stale candidate stopped the session: %v", err) } } + +func TestCaptureProfileFailureLogsFixedStageWithoutPrivateError(t *testing.T) { + previous := slog.Default() + t.Cleanup(func() { slog.SetDefault(previous) }) + var output bytes.Buffer + slog.SetDefault(slog.New(slog.NewJSONHandler(&output, &slog.HandlerOptions{Level: slog.LevelDebug}))) + failure := errors.New("private capture path and window title") + profile := nativecapture.VideoProfile{Width: 1280, Height: 720, Framerate: 30, Bitrate: 3_000_000} + if captureProfileFailure(t.Context(), profile, "wait-timeout", failure) != failure { + t.Fatal("diagnostic changed the failure result") + } + var record map[string]any + if err := json.Unmarshal(output.Bytes(), &record); err != nil { + t.Fatal(err) + } + if record["event"] != "capture-profile-rejected" || record["stage"] != "wait-timeout" || + record["width"] != float64(1280) || record["height"] != float64(720) || + record["fps"] != float64(30) || record["bitrate"] != float64(3_000_000) || strings.Contains(output.String(), "private") { + t.Fatalf("profile diagnostic changed or leaked: %s", output.String()) + } +} diff --git a/internal/client/nativeviewer/session.go b/internal/client/nativeviewer/session.go index 84ecd43..fbab181 100644 --- a/internal/client/nativeviewer/session.go +++ b/internal/client/nativeviewer/session.go @@ -7,6 +7,7 @@ import ( "time" "github.com/TNTcraftHIM/Screener/internal/client/mediaedge" + "github.com/TNTcraftHIM/Screener/internal/client/nativecapture" "github.com/pion/webrtc/v4" ) @@ -27,6 +28,7 @@ type Options struct { EdgeCapacity int PortMapping bool Events chan<- Event + Relay *mediaedge.RelayOptions } type sessionEdge struct { @@ -43,10 +45,12 @@ type Session struct { events chan<- Event ctx context.Context cancel context.CancelFunc + relay *mediaedge.RelayOptions mu sync.Mutex receivers map[string]*mediaedge.Receiver edges map[string]sessionEdge + profile *nativecapture.VideoProfile closed bool } @@ -68,7 +72,7 @@ func Start(parent context.Context, options Options) (*Session, error) { ctx, cancel := context.WithCancel(parent) session := &Session{ shareID: options.ShareID, edgeCapacity: options.EdgeCapacity, - engine: engine, events: options.Events, ctx: ctx, cancel: cancel, + engine: engine, events: options.Events, ctx: ctx, cancel: cancel, relay: options.Relay, receivers: make(map[string]*mediaedge.Receiver), edges: make(map[string]sessionEdge), } @@ -78,6 +82,28 @@ func Start(parent context.Context, options Options) (*Session, error) { func (session *Session) ShareID() string { return session.shareID } +// UpdateProfile records the Host ceiling; receiving alone never starts derivation. +func (session *Session) UpdateProfile(profile nativecapture.VideoProfile) error { + if !profile.Valid() { + return errors.New("native Viewer profile is invalid") + } + session.mu.Lock() + defer session.mu.Unlock() + if session.closed { + return errors.New("native Viewer session is closed") + } + if session.profile != nil && *session.profile == profile { + return nil + } + for _, receiver := range session.receivers { + if err := receiver.Source().SetRelayProfile(profile); err != nil { + return err + } + } + session.profile = &profile + return nil +} + func (session *Session) AcceptOffer( connectionID string, offer webrtc.SessionDescription, @@ -97,6 +123,7 @@ func (session *Session) AcceptOffer( Offer: offer, ICEServers: iceServers, EdgeCapacity: session.edgeCapacity, + Relay: session.relay, Events: mediaedge.ReceiverEvents{ LocalCandidate: func(candidate *webrtc.ICECandidateInit) { session.emit(Event{ @@ -130,6 +157,13 @@ func (session *Session) AcceptOffer( _ = receiver.Close() return webrtc.SessionDescription{}, false, "", errors.New("native Viewer receiver is unavailable") } + if session.profile != nil { + if err = receiver.Source().SetRelayProfile(*session.profile); err != nil { + session.mu.Unlock() + _ = receiver.Close() + return webrtc.SessionDescription{}, false, "", err + } + } session.receivers[connectionID] = receiver session.mu.Unlock() return answer, receiver.HasAudio(), receiver.Codec(), nil @@ -176,6 +210,10 @@ func (session *Session) prepareEdge( session.mu.Unlock() return webrtc.SessionDescription{}, errors.New("native Viewer edge is unavailable") } + if !local && session.profile == nil { + session.mu.Unlock() + return webrtc.SessionDescription{}, errors.New("native relay requires the Host profile") + } session.mu.Unlock() edge, err := session.engine.NewEdge(receiver.Source(), mediaedge.EdgeOptions{ ConnectionID: connectionID, diff --git a/internal/client/nativeviewer/session_test.go b/internal/client/nativeviewer/session_test.go new file mode 100644 index 0000000..7bc7495 --- /dev/null +++ b/internal/client/nativeviewer/session_test.go @@ -0,0 +1,28 @@ +package nativeviewer + +import ( + "testing" + + "github.com/TNTcraftHIM/Screener/internal/client/nativecapture" +) + +func TestViewerProfileHasNoImplicitCaptureDefault(t *testing.T) { + session := &Session{} + if session.profile != nil || session.UpdateProfile(nativecapture.VideoProfile{}) == nil { + t.Fatal("receive-only Viewer acquired an implicit or invalid profile") + } + profile := nativecapture.VideoProfile{ + Width: 1920, Height: 1080, Framerate: 30, Bitrate: 5_000_000, Preference: "balanced", + } + if err := session.UpdateProfile(profile); err != nil || session.profile == nil || *session.profile != profile { + t.Fatalf("explicit Viewer profile = %v, %v", session.profile, err) + } + previous := session.profile + if err := session.UpdateProfile(profile); err != nil || session.profile != previous { + t.Fatal("unchanged ceiling created another profile update") + } + session.closed = true + if session.UpdateProfile(profile) == nil { + t.Fatal("closed Viewer accepted a profile") + } +} diff --git a/internal/diagnostics/recorder.go b/internal/diagnostics/recorder.go new file mode 100644 index 0000000..61654ba --- /dev/null +++ b/internal/diagnostics/recorder.go @@ -0,0 +1,271 @@ +// Package diagnostics persists bounded structured logs and exports local runtime diagnostics. +package diagnostics + +import ( + "archive/zip" + "encoding/json" + "errors" + "fmt" + "io" + "log/slog" + "os" + "path/filepath" + "runtime" + "runtime/pprof" + "sync" + "time" +) + +const maxFileBytes = 8 << 20 + +type Recorder struct { + exportMu sync.Mutex + log *rotatingLog + logger *slog.Logger + component string + revision string + started time.Time +} + +// Open appends to the component's current log, retaining one rotated backup. +// Callers own the directory choice and must pass sanitized structured events. +func Open(directory, component, revision string) (*Recorder, error) { + if component != "client" && component != "server" { + return nil, errors.New("diagnostics component must be client or server") + } + if directory == "" { + return nil, errors.New("diagnostics directory is empty") + } + directory, err := filepath.Abs(directory) + if err != nil { + return nil, fmt.Errorf("resolve diagnostics directory: %w", err) + } + if err := os.MkdirAll(directory, 0700); err != nil { + return nil, fmt.Errorf("create diagnostics directory: %w", err) + } + path := filepath.Join(directory, component+".log") + for _, name := range []string{path, path + ".1"} { + if err := checkLogFile(name); err != nil { + return nil, err + } + } + file, err := os.OpenFile(path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + return nil, fmt.Errorf("open diagnostics log: %w", err) + } + info, err := file.Stat() + if err != nil { + return nil, errors.Join(err, file.Close()) + } + log := &rotatingLog{file: file, path: path, size: info.Size()} + return &Recorder{ + log: log, logger: slog.New(slog.NewJSONHandler(log, &slog.HandlerOptions{Level: slog.LevelDebug})), + component: component, revision: revision, started: time.Now(), + }, nil +} + +func (r *Recorder) Logger() *slog.Logger { return r.logger } +func (r *Recorder) LogPath() string { return r.log.path } + +// Export samples this process without forcing GC or collecting raw memory, +// configuration, environment variables, media, or arbitrary directory contents. +func (r *Recorder) Export() (path string, returnedErr error) { + r.exportMu.Lock() + defer r.exportMu.Unlock() + logs, err := r.log.snapshot() + if err != nil { + return "", fmt.Errorf("snapshot diagnostics logs: %w", err) + } + now := time.Now() + var memory runtime.MemStats + runtime.ReadMemStats(&memory) + file, err := os.CreateTemp(filepath.Dir(r.LogPath()), r.component+"-diagnostics-"+now.UTC().Format("20060102T150405Z")+"-*.zip") + if err != nil { + return "", fmt.Errorf("create diagnostics archive: %w", err) + } + archive := zip.NewWriter(file) + defer func() { + returnedErr = errors.Join(returnedErr, archive.Close(), file.Sync(), file.Close()) + if returnedErr != nil { + returnedErr = errors.Join(returnedErr, os.Remove(file.Name())) + path = "" + } + }() + for _, log := range logs { + entry, err := archive.Create(log.name) + if err != nil { + return "", err + } + if _, err := entry.Write(log.data); err != nil { + return "", err + } + } + metadata := struct { + Component string `json:"component"` + Revision string `json:"revision"` + GOOS string `json:"goos"` + GOARCH string `json:"goarch"` + GoVersion string `json:"goVersion"` + CollectedAt time.Time `json:"collectedAt"` + UptimeSeconds float64 `json:"uptimeSeconds"` + }{r.component, r.revision, runtime.GOOS, runtime.GOARCH, runtime.Version(), now.UTC(), now.Sub(r.started).Seconds()} + for _, item := range []struct { + name string + value any + }{{"metadata.json", metadata}, {"memory.json", memory}} { + entry, err := archive.Create(item.name) + if err != nil { + return "", err + } + if err := json.NewEncoder(entry).Encode(item.value); err != nil { + return "", err + } + } + for _, profile := range []struct { + name string + file string + debug int + }{{"heap", "heap.pprof", 0}, {"goroutine", "goroutines.txt", 1}} { + entry, err := archive.Create(profile.file) + if err != nil { + return "", err + } + if err := pprof.Lookup(profile.name).WriteTo(&boundedWriter{writer: entry, remaining: maxFileBytes}, profile.debug); err != nil { + return "", fmt.Errorf("write %s profile: %w", profile.name, err) + } + } + return file.Name(), nil +} + +// Close waits for an export already in progress. It is safe to call repeatedly. +func (r *Recorder) Close() error { + r.exportMu.Lock() + defer r.exportMu.Unlock() + r.log.mu.Lock() + defer r.log.mu.Unlock() + if r.log.file != nil { + r.log.err = errors.Join(r.log.err, r.log.file.Sync(), r.log.file.Close()) + r.log.file = nil + } + return r.log.err +} + +type rotatingLog struct { + mu sync.Mutex + file *os.File + path string + size int64 + err error +} + +func (l *rotatingLog) Write(data []byte) (n int, err error) { + l.mu.Lock() + defer l.mu.Unlock() + if l.file == nil { + return 0, os.ErrClosed + } + // slog discards handler errors; retain the first failure for Export and Close. + defer func() { + if err != nil && l.err == nil { + l.err = err + } + }() + if len(data) > maxFileBytes { + return 0, errors.New("diagnostics log record exceeds 8 MiB") + } + if l.size+int64(len(data)) > maxFileBytes { + if err := l.rotate(); err != nil { + return 0, err + } + } + n, err = l.file.Write(data) + l.size += int64(n) + return n, err +} + +func (l *rotatingLog) rotate() error { + err := l.file.Close() + l.file = nil + if err != nil { + return err + } + if err := os.Remove(l.path + ".1"); err != nil && !errors.Is(err, os.ErrNotExist) { + return err + } + if err := os.Rename(l.path, l.path+".1"); err != nil { + return err + } + l.file, err = os.OpenFile(l.path, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + l.size = 0 + return err +} + +type logSnapshot struct { + name string + data []byte +} + +func (l *rotatingLog) snapshot() ([]logSnapshot, error) { + l.mu.Lock() + defer l.mu.Unlock() + if l.err != nil { + return nil, l.err + } + if l.file == nil { + return nil, os.ErrClosed + } + if err := l.file.Sync(); err != nil { + return nil, err + } + // ponytail: at most 16 MiB copied under the log lock; compression runs outside it. + var logs []logSnapshot + for _, path := range []string{l.path + ".1", l.path} { + if err := checkLogFile(path); err != nil { + return nil, err + } + file, err := os.Open(path) + if errors.Is(err, os.ErrNotExist) && path != l.path { + continue + } + if err != nil { + return nil, err + } + data, readErr := io.ReadAll(io.LimitReader(file, maxFileBytes+1)) + if err := errors.Join(readErr, file.Close()); err != nil { + return nil, err + } + if len(data) > maxFileBytes { + return nil, errors.New("diagnostics log exceeds 8 MiB") + } + logs = append(logs, logSnapshot{filepath.Base(path), data}) + } + return logs, nil +} + +func checkLogFile(path string) error { + info, err := os.Lstat(path) + if errors.Is(err, os.ErrNotExist) { + return nil + } + if err != nil { + return err + } + if !info.Mode().IsRegular() || info.Size() > maxFileBytes { + return fmt.Errorf("diagnostics log %q must be a regular file no larger than 8 MiB", path) + } + return nil +} + +type boundedWriter struct { + writer io.Writer + remaining int +} + +func (w *boundedWriter) Write(data []byte) (int, error) { + if len(data) > w.remaining { + return 0, errors.New("diagnostics profile exceeds 8 MiB") + } + n, err := w.writer.Write(data) + w.remaining -= n + return n, err +} diff --git a/internal/diagnostics/recorder_test.go b/internal/diagnostics/recorder_test.go new file mode 100644 index 0000000..2b7851e --- /dev/null +++ b/internal/diagnostics/recorder_test.go @@ -0,0 +1,184 @@ +package diagnostics + +import ( + "archive/zip" + "bytes" + "compress/gzip" + "encoding/json" + "errors" + "io" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" +) + +func TestRecorderRotationAndExport(t *testing.T) { + directory := t.TempDir() + const private = "private-environment-and-media-sentinel" + t.Setenv("SCREENER_DIAGNOSTICS_TEST_SECRET", private) + if err := os.WriteFile(filepath.Join(directory, "private-config.json"), []byte(private), 0600); err != nil { + t.Fatal(err) + } + recorder, err := Open(directory, "client", "test-revision") + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = recorder.Close() }) + chunk := bytes.Repeat([]byte("x"), maxFileBytes/2) + for range 5 { + if _, err := recorder.log.Write(chunk); err != nil { + t.Fatal(err) + } + } + recorder.Logger().Debug("capture state", "event", "started") + for _, path := range []string{recorder.LogPath(), recorder.LogPath() + ".1"} { + info, err := os.Stat(path) + if err != nil || info.Size() > maxFileBytes { + t.Fatalf("bounded log %s: %v, %v", path, info, err) + } + } + if _, err := os.Stat(recorder.LogPath() + ".2"); !errors.Is(err, os.ErrNotExist) { + t.Fatalf("unexpected second backup: %v", err) + } + archivePath, err := recorder.Export() + if err != nil { + t.Fatal(err) + } + archive, err := zip.OpenReader(archivePath) + if err != nil { + t.Fatal(err) + } + defer archive.Close() + expected := map[string]bool{"client.log": false, "client.log.1": false, "metadata.json": false, "memory.json": false, "heap.pprof": false, "goroutines.txt": false} + for _, file := range archive.File { + if _, ok := expected[file.Name]; !ok { + t.Fatalf("unexpected archive entry: %s", file.Name) + } + expected[file.Name] = true + entry, err := file.Open() + if err != nil { + t.Fatal(err) + } + data, readErr := io.ReadAll(entry) + if err := errors.Join(readErr, entry.Close()); err != nil { + t.Fatal(err) + } + if bytes.Contains(data, []byte(private)) { + t.Fatalf("private data in %s", file.Name) + } + switch file.Name { + case "client.log": + if !bytes.Contains(data, []byte(`"level":"DEBUG"`)) { + t.Fatal("debug event was not persisted") + } + case "metadata.json": + var metadata map[string]any + if err := json.Unmarshal(data, &metadata); err != nil { + t.Fatal(err) + } + if metadata["revision"] != "test-revision" || metadata["component"] != "client" || metadata["goos"] != runtime.GOOS || len(metadata) != 7 { + t.Fatalf("unexpected metadata: %v", metadata) + } + case "memory.json": + var memory runtime.MemStats + if err := json.Unmarshal(data, &memory); err != nil || memory.Sys == 0 { + t.Fatalf("invalid memory counters: %v", err) + } + case "heap.pprof": + profile, err := gzip.NewReader(bytes.NewReader(data)) + if err != nil { + t.Fatal(err) + } + profileData, err := io.ReadAll(profile) + if err != nil || len(profileData) == 0 || bytes.Contains(profileData, []byte(private)) { + t.Fatalf("invalid or private heap allocation profile: %v", err) + } + if err := profile.Close(); err != nil { + t.Fatal(err) + } + case "goroutines.txt": + if !bytes.HasPrefix(data, []byte("goroutine profile: total ")) || bytes.Contains(data, []byte("[running]:")) { + t.Fatal("expected aggregated goroutine profile") + } + } + } + for name, found := range expected { + if !found { + t.Fatalf("missing archive entry: %s", name) + } + } + second, err := recorder.Export() + if err != nil || second == archivePath { + t.Fatalf("export must have a unique path: %q, %v", second, err) + } +} + +func TestRecorderCloseAndFailures(t *testing.T) { + if _, err := Open(t.TempDir(), "../client", "test"); err == nil { + t.Fatal("accepted a component path") + } + recorder, err := Open(t.TempDir(), "server", "test") + if err != nil { + t.Fatal(err) + } + var workers sync.WaitGroup + var started sync.WaitGroup + started.Add(4) + stop := make(chan struct{}) + defer func() { + close(stop) + workers.Wait() + }() + for worker := range 4 { + workers.Go(func() { + recorder.Logger().Debug("concurrent log", "worker", worker) + started.Done() + for { + select { + case <-stop: + return + default: + } + recorder.Logger().Debug("concurrent log", "worker", worker) + } + }) + } + started.Wait() + if _, err := recorder.Export(); err != nil { + t.Fatal(err) + } + workers.Go(func() { + if _, err := recorder.Export(); err != nil && !errors.Is(err, os.ErrClosed) { + t.Errorf("concurrent export: %v", err) + } + }) + if err := recorder.Close(); err != nil { + t.Fatal(err) + } + if err := recorder.Close(); err != nil { + t.Fatal(err) + } + if _, err := recorder.Export(); !errors.Is(err, os.ErrClosed) { + t.Fatalf("export after close: %v", err) + } + failing, err := Open(t.TempDir(), "client", "test") + if err != nil { + t.Fatal(err) + } + if err := failing.log.file.Close(); err != nil { + t.Fatal(err) + } + failing.Logger().Error("write fails") + if _, err := failing.Export(); !errors.Is(err, os.ErrClosed) { + t.Fatalf("export hid a log write failure: %v", err) + } + if err := failing.Close(); !errors.Is(err, os.ErrClosed) { + t.Fatalf("close hid a log write failure: %v", err) + } + if _, err := (&boundedWriter{writer: io.Discard, remaining: 1}).Write([]byte("xx")); err == nil || !strings.Contains(err.Error(), "8 MiB") { + t.Fatalf("profile output limit: %v", err) + } +} diff --git a/internal/media/encoded/packetizer.go b/internal/media/encoded/packetizer.go new file mode 100644 index 0000000..8e31640 --- /dev/null +++ b/internal/media/encoded/packetizer.go @@ -0,0 +1,87 @@ +// Package encoded contains shared encoded-video primitives, independent of +// room authority, capture implementation and PeerConnection lifecycle. +package encoded + +import ( + "errors" + "time" + + "github.com/pion/rtp" +) + +const MaxAccessUnitBytes = 4 * 1024 * 1024 + +// At the 1200-byte media MTU, this window covers a maximum-sized fragmented AU +// with header headroom. It does not promise unlimited tiny-NAL packetization. +const MaxPacketWindow = MaxAccessUnitBytes / 1024 +const videoClockRate = 90_000 + +var ErrInvalidTimestamp = errors.New("encoded video timestamp is not monotonic") + +// Frame is one complete codec access unit. Recovery includes the codec +// configuration needed to decode independently of previous output. +type Frame struct { + Data []byte + PTS, Duration time.Duration + Recovery bool +} + +// Packetizer preserves one outbound RTP sequence and source timeline across +// complete access units. Its owner serializes calls; it buffers no media. +type Packetizer struct { + rtp rtp.Packetizer + hasTimestamp bool + lastTimestamp time.Duration + timestampRemainder float64 + rebasePending bool + rebaseInput time.Duration + rebaseOutput time.Duration +} + +func NewPacketizer(payloader rtp.Payloader, payloadType uint8, mtu uint16) *Packetizer { + return &Packetizer{rtp: rtp.NewPacketizer( + mtu, payloadType, 0, payloader, rtp.NewRandomSequencer(), videoClockRate, + )} +} + +func (p *Packetizer) Packetize(data []byte, timestamp, duration time.Duration) ([]*rtp.Packet, error) { + if len(data) == 0 || len(data) > MaxAccessUnitBytes || timestamp < 0 || duration <= 0 { + return nil, errors.New("encoded video access unit is invalid") + } + if p.rebasePending { + p.rebasePending = false + p.rebaseInput = timestamp + p.rebaseOutput = p.lastTimestamp + duration + timestamp = p.rebaseOutput + } else if p.rebaseOutput > 0 { + if timestamp < p.rebaseInput { + return nil, ErrInvalidTimestamp + } + timestamp = p.rebaseOutput + timestamp - p.rebaseInput + } + if p.hasTimestamp { + if timestamp <= p.lastTimestamp { + return nil, ErrInvalidTimestamp + } + ticksFloat := (timestamp-p.lastTimestamp).Seconds()*videoClockRate + p.timestampRemainder + ticks := uint64(ticksFloat) + p.timestampRemainder = ticksFloat - float64(ticks) + p.rtp.SkipSamples(uint32(ticks)) + } else { + p.hasTimestamp = true + } + p.lastTimestamp = timestamp + packets := p.rtp.Packetize(data, 0) + if len(packets) == 0 { + return nil, errors.New("encoded video access unit produced no RTP packets") + } + return packets, nil +} + +// BeginGeneration rebases a replacement source clock. Layer changes within the +// same source must not call it: they retain the original source timestamps. +func (p *Packetizer) BeginGeneration() { + p.rebasePending = true + p.rebaseInput = 0 + p.rebaseOutput = 0 +} diff --git a/internal/media/encoded/packetizer_test.go b/internal/media/encoded/packetizer_test.go new file mode 100644 index 0000000..7a5d981 --- /dev/null +++ b/internal/media/encoded/packetizer_test.go @@ -0,0 +1,84 @@ +package encoded + +import ( + "bytes" + "errors" + "testing" + "time" + + "github.com/pion/rtp/codecs" +) + +func TestPacketizerKeepsVP8IdentityAcrossAccessUnits(t *testing.T) { + p := NewPacketizer(&codecs.VP8Payloader{EnablePictureID: true}, 96, 1200) + var sequence uint16 + var timestamp uint32 + var pictureID uint16 + for frame := 0; frame < 4; frame++ { + data := bytes.Repeat([]byte{byte(frame)}, 1000+frame*1100) + packets, err := p.Packetize(data, time.Duration(frame)*time.Second/30, time.Second/30) + if err != nil { + t.Fatal(err) + } + var decoded []byte + for index, packet := range packets { + var header codecs.VP8Packet + payload, err := header.Unmarshal(packet.Payload) + if err != nil { + t.Fatal(err) + } + decoded = append(decoded, payload...) + if (frame > 0 || index > 0) && packet.SequenceNumber != sequence+1 { + t.Fatal("sequence changed between encoded representations") + } + if index == 0 && frame > 0 { + if delta := packet.Timestamp - timestamp; delta < 2999 || delta > 3001 { + t.Fatalf("source clock delta = %d", delta) + } + if header.PictureID != pictureID+1 { + t.Fatal("VP8 picture identity restarted") + } + } + if packet.Marker != (index == len(packets)-1) { + t.Fatal("access unit marker did not match frame boundary") + } + sequence, timestamp, pictureID = packet.SequenceNumber, packet.Timestamp, header.PictureID + } + if !bytes.Equal(data, decoded) { + t.Fatal("packetization changed the shared encoded payload") + } + } + if _, err := p.Packetize([]byte{1}, 0, time.Second/30); !errors.Is(err, ErrInvalidTimestamp) { + t.Fatalf("backward source timestamp error = %v", err) + } + p.BeginGeneration() + packets, err := p.Packetize([]byte{1}, 0, time.Second/30) + if err != nil || len(packets) != 1 || packets[0].SequenceNumber != sequence+1 { + t.Fatalf("source generation did not retain packet identity: %v", err) + } +} + +// This measures RTP bookkeeping only, never repeated video encoding. +func BenchmarkSharedFramePacketization(b *testing.B) { + data := make([]byte, 20*1024) + for _, children := range []int{1, 2} { + name := "one-packetizer-reference" + if children == 2 { + name = "two-independent-output-identities" + } + b.Run(name, func(b *testing.B) { + packetizers := make([]*Packetizer, children) + for i := range packetizers { + packetizers[i] = NewPacketizer(&codecs.VP8Payloader{EnablePictureID: true}, 96, 1200) + } + b.ReportAllocs() + for i := 0; i < b.N; i++ { + for _, packetizer := range packetizers { + if _, err := packetizer.Packetize(data, time.Duration(i)*time.Second/30, time.Second/30); err != nil { + b.Fatal(err) + } + } + } + }) + } +} diff --git a/internal/media/forwarding/encoded_source.go b/internal/media/forwarding/encoded_source.go new file mode 100644 index 0000000..48001ee --- /dev/null +++ b/internal/media/forwarding/encoded_source.go @@ -0,0 +1,125 @@ +package forwarding + +import ( + "errors" + "math" + "time" + + "github.com/TNTcraftHIM/Screener/internal/media/encoded" + "github.com/livekit/mediatransportutil" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/utils/mono" + "github.com/pion/rtp" + "github.com/pion/rtp/codecs" + "github.com/pion/webrtc/v4" +) + +// EncodedSource feeds complete codec outputs into the same forwarding buffers +// used for received RTP. Its capture/decoder owner serializes calls and fences +// replacement input; no network, capture or wall-clock scheduler lives here. +type EncodedSource struct { + *Source + packetizers []*encoded.Packetizer + packets []uint32 + octets []uint64 + codecBytes []uint64 + rtpBytes []uint64 + inputPTS time.Duration + anchorPTS time.Duration + anchorTime time.Time + hasInput bool +} + +func NewEncodedSource(options SourceOptions) (*EncodedSource, error) { + source, err := NewSource(options) + if err != nil { + return nil, err + } + input := &EncodedSource{ + Source: source, packetizers: make([]*encoded.Packetizer, len(options.Formats)), + packets: make([]uint32, len(options.Formats)), octets: make([]uint64, len(options.Formats)), + codecBytes: make([]uint64, len(options.Formats)), rtpBytes: make([]uint64, len(options.Formats)), + } + for layer := range input.packetizers { + var payloader rtp.Payloader = &codecs.H264Payloader{} + if options.Codec.MimeType == webrtc.MimeTypeVP8 { + payloader = &codecs.VP8Payloader{EnablePictureID: true} + } + input.packetizers[layer] = encoded.NewPacketizer(payloader, uint8(options.Codec.PayloadType), 1200) + // These identities stay inside this source's private buffers. DownTrack + // supplies the independently negotiated wire identity for every child. + if err = source.BindLayer(layer, uint32(layer+1), webrtc.RTPParameters{ + Codecs: []webrtc.RTPCodecParameters{options.Codec}, HeaderExtensions: options.HeaderExtensions, + }); err != nil { + source.Close() + return nil, err + } + } + return input, nil +} + +func (input *EncodedSource) BeginFrame(pts time.Duration, at time.Time) error { + if input.IsClosed() { + return errors.New("encoded forwarding source is closed") + } + if pts < 0 || at.IsZero() || input.hasInput && pts <= input.inputPTS { + return encoded.ErrInvalidTimestamp + } + if !input.hasInput { + input.anchorPTS, input.anchorTime = pts, at + } + input.hasInput, input.inputPTS = true, pts + return nil +} + +func (input *EncodedSource) WriteFrame(layer int, frame encoded.Frame) error { + if layer < 0 || layer >= len(input.packetizers) || !input.hasInput || + frame.PTS < input.anchorPTS || frame.PTS > input.inputPTS { + return errors.New("encoded output has no matching source input") + } + packets, err := input.packetizers[layer].Packetize(frame.Data, frame.PTS, frame.Duration) + if err != nil { + return err + } + for _, packet := range packets { + packet.SSRC = uint32(layer + 1) + if err = input.Source.WriteRTP(layer, packet); err != nil { + return err + } + input.packets[layer]++ + input.octets[layer] += uint64(len(packet.Payload)) + input.rtpBytes[layer] += uint64(packet.MarshalSize()) + } + input.codecBytes[layer] += uint64(len(frame.Data)) + // Correlate each layer with the actual input clock, not its encoder finish + // time. This is internal correlation data, not a backdated wire sender report. + return input.SetCorrelation(layer, &livekit.RTCPSenderReportState{ + RtpTimestamp: packets[0].Timestamp, + NtpTimestamp: uint64(mediatransportutil.ToNtpTime(input.anchorTime.Add(frame.PTS - input.anchorPTS))), + Packets: input.packets[layer], Octets: input.octets[layer], At: mono.UnixNano(), + }) +} + +// CodecBudget converts source-RTP allocation units to encoder AU-payload units. +// Actual packetization includes codec descriptors; bitrate comes from the +// existing tracker window so dormant time does not dilute the overhead rate. +// Downstream extensions and SRTP/UDP overhead are outside this source measure. +func (input *EncodedSource) CodecBudget(layer int, rtpBudget int64) int64 { + if layer < 0 || layer >= len(input.rtpBytes) || input.rtpBytes[layer] <= input.codecBytes[layer] { + return max(rtpBudget, 0) + } + _, rates := input.GetLayeredBitrate() + overhead := float64(rates[layer][0]) * float64(input.rtpBytes[layer]-input.codecBytes[layer]) / float64(input.rtpBytes[layer]) + return max(rtpBudget-int64(math.Ceil(overhead)), 0) +} + +func (input *EncodedSource) BeginGeneration() { + // Capture replacement changes the input clock, not these RTP identities. + // Rebase the packetizers; restarting DownTrack as well would rebase twice. + for _, packetizer := range input.packetizers { + packetizer.BeginGeneration() + } + input.hasInput = false + clear(input.codecBytes) + clear(input.rtpBytes) +} diff --git a/internal/media/forwarding/encoded_source_test.go b/internal/media/forwarding/encoded_source_test.go new file mode 100644 index 0000000..aab0f97 --- /dev/null +++ b/internal/media/forwarding/encoded_source_test.go @@ -0,0 +1,68 @@ +package forwarding + +import ( + "math" + "testing" + "time" + + "github.com/TNTcraftHIM/Screener/internal/media/encoded" + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/pion/webrtc/v4" +) + +func TestCodecBudgetUsesActualSourceFraming(t *testing.T) { + codec := webrtc.RTPCodecParameters{RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeVP8, ClockRate: 90_000, + }, PayloadType: 96} + input, err := NewEncodedSource(SourceOptions{ID: "framing", StreamID: "framing", Codec: codec, + Formats: []LayerFormat{{Width: 8, Height: 8, Bitrate: 100_000}, {Width: 16, Height: 16, Bitrate: 1_000_000}}}) + if err != nil { + t.Fatal(err) + } + defer input.Close() + if input.CodecBudget(0, 120_000) != 120_000 { + t.Fatal("cold source invented packet overhead") + } + period := time.Second / 30 + ticker := time.NewTicker(period) + defer ticker.Stop() + deadline := time.NewTimer(6 * time.Second) + defer deadline.Stop() + var rates sfu.Bitrates + for frame := 0; rates[0][0] == 0 || rates[1][0] == 0; frame++ { + select { + case <-ticker.C: + case <-deadline.C: + t.Fatal("existing screen-share measurement window produced no bitrate") + } + pts := time.Duration(frame) * period + if err = input.BeginFrame(pts, time.Now()); err != nil { + t.Fatal(err) + } + for layer, size := range []int{80, 4000} { + data := make([]byte, size) + copy(data, sfu.VP8KeyFrame8x8) + if err = input.WriteFrame(layer, encoded.Frame{Data: data, PTS: pts, Duration: period}); err != nil { + t.Fatal(err) + } + } + _, rates = input.GetLayeredBitrate() + } + for layer := range input.rtpBytes { + if input.rtpBytes[layer]-input.codecBytes[layer] <= uint64(input.packets[layer])*12 { + t.Fatal("codec descriptors were omitted from framing cost") + } + overhead := int64(math.Ceil(float64(rates[layer][0]) * float64(input.rtpBytes[layer]-input.codecBytes[layer]) / float64(input.rtpBytes[layer]))) + if got := input.CodecBudget(layer, 120_000); got != max(120_000-overhead, 0) || got >= 120_000 { + t.Fatalf("layer %d codec budget=%d overhead=%d", layer, got, overhead) + } + } + input.codecBytes[0] = input.rtpBytes[0] + 1 // Annex-B delimiters can be removed by packetization. + if input.CodecBudget(0, 120_000) != 120_000 { + t.Fatal("removed codec framing increased the supplied budget") + } + input.BeginGeneration() + if input.CodecBudget(1, 120_000) != 120_000 || input.codecBytes[1] != 0 || input.rtpBytes[1] != 0 { + t.Fatal("replacement source inherited stale framing ratios") + } +} diff --git a/internal/media/forwarding/output.go b/internal/media/forwarding/output.go new file mode 100644 index 0000000..000f92b --- /dev/null +++ b/internal/media/forwarding/output.go @@ -0,0 +1,389 @@ +// Package forwarding adapts mature RTP forwarding components without owning +// room authority, capture or a second congestion controller. +package forwarding + +import ( + "io" + "slices" + "sync" + "sync/atomic" + + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/protocol/livekit" + "github.com/pion/webrtc/v4" +) + +// Output owns allocation for one video stream on one child connection. Its +// physical owner closes it; budget changes and upstream events do not create +// another scheduler. Do not attach a StreamAllocator to the same DownTrack. +type Output struct { + *sfu.DownTrack + mu sync.Mutex + writeMu sync.Mutex + budget int64 + current int32 + allocation sfu.VideoAllocation + hasAllocation bool + closed bool + onActivity func() + onDemandChanged func() +} + +// Snapshot is read by the parent at its input-frame boundary. Current means +// accepted for forwarding, not remotely decoded. Prepare admits codec work, +// not network delivery, from the latest supplied budget (initial or observed). +// Its caller gates observed-only decisions with Transport.TargetBitrate. +// -1 means no selected layer. +type Snapshot struct { + Prepare, Target, Requested, Current int32 + Paused bool + // VideoBudget is the current connection estimate after audio reservation. + // It is codec input, not a claim that this bitrate is available or delivered. + VideoBudget int64 +} + +func NewOutput(track *sfu.DownTrack, budget int64) *Output { + return &Output{DownTrack: track, budget: max(budget, 0), current: -1} +} + +func (output *Output) SetBudget(budget int64) { + output.mu.Lock() + defer output.mu.Unlock() + output.budget = max(budget, 0) + output.reconcileLocked() +} + +func (output *Output) Reconcile() { + output.mu.Lock() + defer output.mu.Unlock() + output.reconcileLocked() +} + +func (output *Output) reconcileLocked() { + defer output.notifyDemand() + if output.closed { + return + } + if output.budget == 0 { + output.allocation = output.Pause() + output.hasAllocation = true + return + } + available, rates := output.Receiver().GetLayeredBitrate() + measured := false + for _, layer := range available { + if rates[layer][0] > 0 { + measured = true + break + } + } + if !measured { + if len(available) > 0 { + // A real layer can deliver before its bitrate window exists. The + // library bootstrap also establishes the target for PLI recovery. + output.allocation = output.AllocateOptimal(false, false) + output.hasAllocation = true + } + return + } + output.ProvisionalAllocatePrepare() + remaining := output.budget + maximum := output.MaxLayer() + controlled, ok := output.Receiver().(interface{ RateControlled(int32) bool }) + keepLowest := ok && controlled.RateControlled(0) && slices.Contains(available, int32(0)) + for spatial := int32(0); spatial <= maximum.Spatial; spatial++ { + for temporal := int32(0); temporal <= maximum.Temporal; temporal++ { + _, used := output.ProvisionalAllocate(remaining, + buffer.VideoLayer{Spatial: spatial, Temporal: temporal}, !(keepLowest && spatial == 0), false) + remaining -= used + } + } + // Only a live local encoder can follow this child's budget below the current + // lowest-layer measurement; raw forwarding retains the library pause policy. + output.allocation = output.ProvisionalAllocateCommit() + output.hasAllocation = true +} + +func (output *Output) State() Snapshot { + output.mu.Lock() + defer output.mu.Unlock() + if output.closed { + return Snapshot{Prepare: -1, Target: -1, Requested: -1, Current: -1, Paused: true} + } + prepare := int32(-1) + receiver := output.Receiver() + layers := buffer.GetVideoLayersForMimeType(receiver.Mime(), receiver.TrackInfo()) + maximum := output.MaxLayer().Spatial + lowest := int32(-1) + for _, layer := range layers { + if layer.SpatialLayer >= 0 && layer.SpatialLayer < int32(len(layers)) && + layer.SpatialLayer <= maximum && layer.Bitrate > 0 { + if lowest < 0 || layer.SpatialLayer < lowest { + lowest = layer.SpatialLayer + } + if int64(layer.Bitrate) <= output.budget { + prepare = max(prepare, layer.SpatialLayer) + } + } + } + // Below the lowest ceiling, keep one codec output eligible for rate control; + // otherwise pausing would retire the output needed to recover delivery. + if prepare < 0 && output.budget > 0 { + prepare = lowest + } + if !output.hasAllocation { + return Snapshot{Prepare: prepare, Target: -1, Requested: -1, Current: output.current, VideoBudget: output.budget} + } + return Snapshot{ + Prepare: prepare, + Target: output.allocation.TargetLayer.Spatial, Requested: output.allocation.RequestLayerSpatial, + Current: output.current, Paused: !output.allocation.TargetLayer.IsValid(), VideoBudget: output.budget, + } +} + +func (output *Output) probeDemand() (int64, int64) { + output.mu.Lock() + defer output.mu.Unlock() + if output.closed || !output.hasAllocation { + return 0, 0 + } + usage := output.BandwidthRequested() + if transition, available := output.GetNextHigherTransition(false); available { + return transition.BandwidthDelta, usage + } + // Dormant ceilings are desired demand only. They never enter the receiver's + // measured bitrate table or authorize forwarding an unavailable layer. + next, desired := int32(buffer.DefaultMaxLayerSpatial+1), int64(0) + for _, layer := range buffer.GetVideoLayersForMimeType(output.Receiver().Mime(), output.Receiver().TrackInfo()) { + if layer.SpatialLayer > output.allocation.TargetLayer.Spatial && layer.SpatialLayer <= output.MaxLayer().Spatial && + layer.SpatialLayer < next && int64(layer.Bitrate) > output.budget { + next, desired = layer.SpatialLayer, int64(layer.Bitrate) + } + } + return max(desired-usage, 0), usage +} + +func (output *Output) WriteRTP(packet *buffer.ExtPacket, layer int32) int32 { + return output.writeRTP(packet, layer, nil) +} + +func (output *Output) writeRTP(packet *buffer.ExtPacket, layer int32, receiver sfu.TrackReceiver) int32 { + if output.onActivity != nil { + output.onActivity() + } + output.writeMu.Lock() + defer output.writeMu.Unlock() + if receiver != nil && output.Receiver() != receiver { + return 0 + } + output.mu.Lock() + closed := output.closed + output.mu.Unlock() + if closed { + return 0 + } + // DownTrack listeners may read State; never hold its mutex during callbacks. + written := output.DownTrack.WriteRTP(packet, layer) + if written > 0 && packet.Packet.Marker { + output.mu.Lock() + if !output.closed { + output.current = layer + } + output.mu.Unlock() + output.notifyDemand() + } + return written +} + +// SetReceiver registers its raw DownTrack; retain the complete output wrapper +// and reject callbacks already in flight from a detached receiver. +type outputReceiver struct { + *Source + output *Output + attached bool +} + +func (receiver *outputReceiver) AddDownTrack(sfu.TrackSender) error { + if !receiver.attached { + return nil + } + sender := &outputSourceSender{Output: receiver.output, receiver: receiver} + sender.registering.Store(true) + err := receiver.Source.AddDownTrack(sender) + sender.registering.Store(false) + return err +} + +type outputSourceSender struct { + *Output + receiver *outputReceiver + registering atomic.Bool +} + +func (sender *outputSourceSender) WriteRTP(packet *buffer.ExtPacket, layer int32) int32 { + return sender.Output.writeRTP(packet, layer, sender.receiver) +} + +func (sender *outputSourceSender) Close() { + sender.withReceiver(sender.Output.Close) +} + +func (sender *outputSourceSender) withReceiver(fn func()) { + // AddDownTrack invokes initial metadata synchronously inside SetReceiver, + // which already holds writeMu. Later source events take the normal lock. + if !sender.registering.Load() { + sender.Output.writeMu.Lock() + defer sender.Output.writeMu.Unlock() + } + if sender.Output.Receiver() == sender.receiver { + fn() + } +} + +func (sender *outputSourceSender) ReceiverRestart(receiver sfu.TrackReceiver) { + sender.withReceiver(func() { sender.Output.receiverRestartLocked(receiver) }) +} + +func (sender *outputSourceSender) UpTrackLayersChange() { + sender.withReceiver(sender.Output.UpTrackLayersChange) +} + +func (sender *outputSourceSender) UpTrackBitrateAvailabilityChange() { + sender.withReceiver(sender.Output.UpTrackBitrateAvailabilityChange) +} + +func (sender *outputSourceSender) UpTrackMaxPublishedLayerChange(layer int32) { + sender.withReceiver(func() { sender.Output.UpTrackMaxPublishedLayerChange(layer) }) +} + +func (sender *outputSourceSender) UpTrackMaxTemporalLayerSeenChange(layer int32) { + sender.withReceiver(func() { sender.Output.UpTrackMaxTemporalLayerSeenChange(layer) }) +} + +func (sender *outputSourceSender) UpTrackBitrateReport(layers []int32, rates sfu.Bitrates) { + sender.withReceiver(func() { sender.Output.UpTrackBitrateReport(layers, rates) }) +} + +func (sender *outputSourceSender) HandleRTCPSenderReportData(payloadType webrtc.PayloadType, layer int32, report *livekit.RTCPSenderReportState) error { + sender.Output.writeMu.Lock() + defer sender.Output.writeMu.Unlock() + if sender.Output.Receiver() != sender.receiver { + return nil + } + return sender.Output.HandleRTCPSenderReportData(payloadType, layer, report) +} + +func (output *Output) replaceReceiver(receiver *outputReceiver) error { + output.writeMu.Lock() + defer output.writeMu.Unlock() + if output.IsClosed() { + return io.ErrClosedPipe + } + // A different source has different RTP/NTP references even with the same + // logical layer SSRCs. Restart clears those references; Resync does not. + output.DownTrack.ReceiverRestart(output.Receiver()) + output.DownTrack.SetReceiver(receiver) + output.mu.Lock() + output.current = -1 + output.hasAllocation = false + output.reconcileLocked() + output.mu.Unlock() + return nil +} + +// BeginFrame fences a controlled Native input before any of its outputs arrive. +// Raw RTP has no such boundary and keeps DownTrack's ordinary switching path. +func (output *Output) BeginFrame() { + output.writeMu.Lock() + defer output.writeMu.Unlock() + output.mu.Lock() + switching := !output.closed && output.hasAllocation && output.current >= 0 && + output.allocation.TargetLayer.IsValid() && output.current != output.allocation.TargetLayer.Spatial + output.mu.Unlock() + if !switching { + return + } + output.DownTrack.Resync() + output.mu.Lock() + output.current = -1 + output.mu.Unlock() +} + +func (output *Output) Resync() { + output.writeMu.Lock() + defer output.writeMu.Unlock() + output.DownTrack.Resync() + output.mu.Lock() + output.current = -1 + output.hasAllocation = false + output.mu.Unlock() +} + +func (output *Output) ReceiverRestart(receiver sfu.TrackReceiver) { + output.writeMu.Lock() + defer output.writeMu.Unlock() + output.receiverRestartLocked(receiver) +} + +func (output *Output) receiverRestartLocked(receiver sfu.TrackReceiver) { + output.DownTrack.ReceiverRestart(receiver) + output.mu.Lock() + output.current = -1 + // ReceiverBase updates availability after forwarding a new input. Arm the + // library's existing acquisition target before that first recovery AU arrives. + if !output.closed { + output.allocation = output.AllocateOptimal(false, false) + output.hasAllocation = true + } + output.mu.Unlock() +} + +func (output *Output) SetMaxSpatialLayer(layer int32) { + output.DownTrack.SetMaxSpatialLayer(layer) + output.Reconcile() +} + +func (output *Output) SetMaxTemporalLayer(layer int32) { + output.DownTrack.SetMaxTemporalLayer(layer) + output.Reconcile() +} + +func (output *Output) UpTrackLayersChange() { + output.DownTrack.UpTrackLayersChange() + output.Reconcile() +} + +func (output *Output) UpTrackBitrateAvailabilityChange() { + output.DownTrack.UpTrackBitrateAvailabilityChange() + output.Reconcile() +} + +func (output *Output) UpTrackMaxPublishedLayerChange(layer int32) { + output.DownTrack.UpTrackMaxPublishedLayerChange(layer) + output.Reconcile() +} + +func (output *Output) UpTrackMaxTemporalLayerSeenChange(layer int32) { + output.DownTrack.UpTrackMaxTemporalLayerSeenChange(layer) + output.Reconcile() +} + +func (output *Output) UpTrackBitrateReport(layers []int32, rates sfu.Bitrates) { + output.DownTrack.UpTrackBitrateReport(layers, rates) + output.Reconcile() +} + +func (output *Output) Close() { + output.mu.Lock() + output.closed = true + output.mu.Unlock() + output.DownTrack.Close() + output.notifyDemand() +} + +func (output *Output) notifyDemand() { + if output.onDemandChanged != nil { + output.onDemandChanged() + } +} diff --git a/internal/media/forwarding/output_test.go b/internal/media/forwarding/output_test.go new file mode 100644 index 0000000..c5ecf79 --- /dev/null +++ b/internal/media/forwarding/output_test.go @@ -0,0 +1,180 @@ +package forwarding + +import ( + "testing" + "time" + + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/pacer" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/pion/webrtc/v4" +) + +// This fixture supplies allocation metadata, not simulated network evidence. +type allocationReceiver struct { + *sfu.ReceiverBase + rates sfu.Bitrates + available []int32 + controlled bool +} + +func (receiver *allocationReceiver) RateControlled(layer int32) bool { + return receiver.controlled && layer == 0 +} + +func (receiver *allocationReceiver) GetLayeredBitrate() ([]int32, sfu.Bitrates) { + return receiver.available, receiver.rates +} + +type outputListener struct{} + +func (*outputListener) OnBindAndConnected() {} +func (*outputListener) OnStatsUpdate(*livekit.AnalyticsStat) {} +func (*outputListener) OnMaxSubscribedLayerChanged(int32) {} +func (*outputListener) OnRttUpdate(uint32) {} +func (*outputListener) OnCodecNegotiated(webrtc.RTPCodecCapability) {} +func (*outputListener) OnDownTrackClose(bool) {} +func (*outputListener) OnStreamStarted(time.Duration) {} + +func TestOutputUsesLibraryAllocationAndExactLifetime(t *testing.T) { + codec := webrtc.RTPCodecParameters{RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeVP8, ClockRate: 90_000, + }, PayloadType: 96} + log := logger.GetLogger() + base := sfu.NewReceiverBase(sfu.ReceiverBaseParams{ + TrackID: "fixture", StreamID: "fixture", Kind: webrtc.RTPCodecTypeVideo, + Codec: codec, Logger: log, StreamTrackerManagerConfig: sfu.DefaultStreamTrackerManagerConfig, + }, &livekit.TrackInfo{Sid: "fixture", Type: livekit.TrackType_VIDEO, + Source: livekit.TrackSource_SCREEN_SHARE, Layers: []*livekit.VideoLayer{ + {SpatialLayer: 0, Bitrate: 100_000}, {SpatialLayer: 1, Bitrate: 300_000}, + {SpatialLayer: 2, Bitrate: 900_000}, + }}, sfu.ReceiverCodecStateNormal) + defer base.Close("test complete", true) + receiver := &allocationReceiver{ReceiverBase: base} + packetPacer := pacer.NewPassThrough(log, nil) + defer packetPacer.Stop() + track, err := sfu.NewDownTrack(sfu.DownTrackParams{ + Codecs: []webrtc.RTPCodecParameters{codec}, Receiver: receiver, + BufferFactory: buffer.NewFactoryOfBufferFactory(64, 16).CreateBufferFactory(), + SubID: "child", StreamID: "fixture", MaxTrack: 64, + Pacer: packetPacer, Logger: log, Listener: &outputListener{}, + }) + if err != nil { + t.Fatal(err) + } + output := NewOutput(track, 1_000_000) + defer output.Close() + output.SetMaxSpatialLayer(2) + output.SetMaxTemporalLayer(0) + output.UpTrackMaxPublishedLayerChange(2) + output.UpTrackMaxTemporalLayerSeenChange(0) + if state := output.State(); state.Target != -1 || state.Current != -1 { + t.Fatalf("unmeasured source pretended to have a target: %+v", state) + } + receiver.available = []int32{0} + output.SetBudget(0) + if state := output.State(); !state.Paused || state.Prepare != -1 || state.VideoBudget != 0 { + t.Fatalf("bootstrap ignored an exhausted video budget: %+v", state) + } + receiver.rates = sfu.Bitrates{{100_000}, {300_000}, {900_000}} + receiver.available = []int32{0, 1, 2} + siblingTrack, err := sfu.NewDownTrack(sfu.DownTrackParams{ + Codecs: []webrtc.RTPCodecParameters{codec}, Receiver: receiver, + BufferFactory: buffer.NewFactoryOfBufferFactory(64, 16).CreateBufferFactory(), + SubID: "healthy", StreamID: "fixture", MaxTrack: 64, + Pacer: packetPacer, Logger: log, Listener: &outputListener{}, + }) + if err != nil { + t.Fatal(err) + } + sibling := NewOutput(siblingTrack, 1_000_000) + defer sibling.Close() + sibling.SetMaxTemporalLayer(0) + sibling.SetMaxSpatialLayer(2) + sibling.UpTrackMaxPublishedLayerChange(2) + sibling.UpTrackMaxTemporalLayerSeenChange(0) + for _, check := range []struct { + budget int64 + layer int32 + }{{1_000_000, 2}, {200_000, 0}, {0, -1}, {400_000, 1}, {1_000_000, 2}} { + output.SetBudget(check.budget) + state := output.State() + if state.Target != check.layer || state.Paused != (check.layer == -1) || state.Current != -1 || state.VideoBudget != check.budget { + t.Fatalf("budget %d: %+v", check.budget, state) + } + } + // A budget below every nominal output still reaches the lowest codec, while + // only actual reduced media may make the library resume its network target. + output.SetBudget(60_000) + if state := output.State(); !state.Paused || state.Target != -1 || state.Prepare != 0 || state.VideoBudget != 60_000 { + t.Fatalf("below-lowest budget cannot request codec adaptation: %+v", state) + } + receiver.controlled = true + output.Reconcile() + if state := output.State(); state.Paused || state.Target != 0 { + t.Fatalf("live lowest codec could not continue bitrate adaptation: %+v", state) + } + output.SetBudget(0) + if state := output.State(); !state.Paused { + t.Fatalf("codec control overrode exhausted connection budget: %+v", state) + } + receiver.available = []int32{1, 2} + receiver.rates[0][0] = 0 + output.SetBudget(60_000) + if state := output.State(); !state.Paused { + t.Fatalf("codec control selected an unavailable lowest layer: %+v", state) + } + receiver.available = []int32{0, 1, 2} + receiver.controlled = false + receiver.rates[0][0] = 50_000 + output.Reconcile() + if state := output.State(); state.Paused || state.Target != 0 || state.VideoBudget != 60_000 { + t.Fatalf("real reduced output did not resume library allocation: %+v", state) + } + sibling.Reconcile() + if state := sibling.State(); state.Paused || state.Target != 2 || state.VideoBudget != 1_000_000 || receiver.rates[2][0] != 900_000 { + t.Fatalf("weak-child adaptation changed the shared high output: %+v", state) + } + receiver.rates[0][0] = 100_000 + // A stopped upper output has no actual bitrate. Admission may prepare it, + // but the library must not select it until it is genuinely available again. + receiver.rates = sfu.Bitrates{{100_000}} + receiver.available = []int32{0} + output.SetBudget(1_000_000) + if state := output.State(); state.Prepare != 2 || state.Target != 0 { + t.Fatalf("stopped output preparation changed real availability: %+v", state) + } + output.SetBudget(200_000) + if state := output.State(); state.Prepare != 0 || state.Target != 0 { + t.Fatalf("low budget prepared an unaffordable upper output: %+v", state) + } + output.SetMaxSpatialLayer(1) + output.SetBudget(1_000_000) + if state := output.State(); state.Prepare != 1 { + t.Fatalf("preparation exceeded child maximum: %+v", state) + } + output.SetMaxSpatialLayer(2) + output.current = 2 // Allocation-only fixture; the real RTP gate verifies emitted frames. + before := output.State() + output.BeginFrame() + after := output.State() + if after.Current != -1 || after.Target != before.Target || after.Requested != before.Requested { + t.Fatalf("controlled frame fence replaced its allocation: before=%+v after=%+v", before, after) + } + output.Resync() + if state := output.State(); state.Target != -1 || state.Current != -1 { + t.Fatalf("source resync retained old output state: %+v", state) + } + output.SetBudget(1_000_000) + output.ReceiverRestart(receiver) + if state := output.State(); state.Current != -1 { + t.Fatalf("receiver restart retained old output state: %+v", state) + } + output.Close() + output.SetBudget(1_000_000) + if state := output.State(); state != (Snapshot{Prepare: -1, Target: -1, Requested: -1, Current: -1, Paused: true}) { + t.Fatalf("closed output revived: %+v", state) + } +} diff --git a/internal/media/forwarding/publication.go b/internal/media/forwarding/publication.go new file mode 100644 index 0000000..f2ac8a6 --- /dev/null +++ b/internal/media/forwarding/publication.go @@ -0,0 +1,636 @@ +package forwarding + +import ( + "crypto/rand" + "errors" + "io" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/bwe" + "github.com/livekit/livekit-server/pkg/sfu/bwe/sendsidebwe" + "github.com/livekit/livekit-server/pkg/sfu/ccutils" + "github.com/livekit/livekit-server/pkg/sfu/pacer" + "github.com/livekit/livekit-server/pkg/sfu/streamallocator" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/pion/interceptor" + "github.com/pion/rtcp" + "github.com/pion/rtp" + "github.com/pion/transport/v4/packetio" + "github.com/pion/webrtc/v4" +) + +const publicationRIDExtension = "urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id" + +// Publication sends existing representations over one simulcast connection. +// LiveKit's allocator owns all track allocations and probes on the shared BWE. +type Publication struct { + PC *webrtc.PeerConnection + source *Source + tracks []*sfu.DownTrack + counters []publicationCounters + audioBytes atomic.Uint64 + allocator *streamallocator.StreamAllocator + bandwidth *publicationBWE + pacer *publicationPacer + mu sync.Mutex + writeMu sync.Mutex + attached bool + closed bool + activeCount int + readers sync.WaitGroup + closeOnce sync.Once + closeErr error +} + +func NewPublication(options TransportOptions) (_ *Publication, err error) { + if options.Source == nil || options.ConnectionID == "" || options.InitialBitrate <= 0 { + return nil, errors.New("publication options are invalid") + } + publication := &Publication{source: options.Source, activeCount: len(options.Source.TrackInfo().Layers)} + publication.counters = make([]publicationCounters, publication.activeCount) + defer func() { + if err != nil { + _ = publication.Close() + } + }() + log := logger.GetDiscardLogger() + publication.bandwidth = &publicationBWE{SendSideBWE: sendsidebwe.NewSendSideBWE(sendsidebwe.SendSideBWEParams{ + Config: sendsidebwe.DefaultSendSideBWEConfig, Logger: log, + })} + publication.bandwidth.target.Store(int64(options.InitialBitrate)) + publication.pacer = &publicationPacer{headers: make(map[uint32]publicationHeaders), boundedPacer: newBoundedPacer( + pacer.NewLeakyBucket(log, nil, 5*time.Millisecond, options.InitialBitrate), log, options.Source.maxPackets, + ), bandwidth: publication.bandwidth} + publication.pacer.sender = pacer.NewBase(log, publication.bandwidth) + publication.bandwidth.pacer = publication.pacer + publication.allocator = streamallocator.NewStreamAllocator(streamallocator.StreamAllocatorParams{ + Config: streamallocator.DefaultStreamAllocatorConfig, BWE: publication.bandwidth, + Pacer: publication.pacer, Logger: log, + }, true, true) + publication.allocator.Start() + publication.pacer.SetBitrate(options.InitialBitrate) + media := &webrtc.MediaEngine{} + codec := options.Source.Codec() + if err = media.RegisterCodec(codec, webrtc.RTPCodecTypeVideo); err != nil { + return nil, err + } + for _, uri := range []string{ + "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01", + "urn:ietf:params:rtp-hdrext:sdes:mid", publicationRIDExtension, + } { + if err = media.RegisterHeaderExtension(webrtc.RTPHeaderExtensionCapability{URI: uri}, webrtc.RTPCodecTypeVideo); err != nil { + return nil, err + } + } + if options.Audio != nil { + audio, ok := options.Audio.(interface { + Codec() webrtc.RTPCodecCapability + }) + if !ok { + return nil, errors.New("publication audio codec is unavailable") + } + if err = media.RegisterCodec(webrtc.RTPCodecParameters{RTPCodecCapability: audio.Codec(), PayloadType: 111}, webrtc.RTPCodecTypeAudio); err != nil { + return nil, err + } + } + factory := buffer.NewFactoryOfBufferFactory(options.Source.maxPackets, buffer.InitPacketBufferSizeAudio).CreateBufferFactory() + settings := options.Settings + settings.BufferFactory = func(kind packetio.BufferPacketType, ssrc uint32) io.ReadWriteCloser { + if kind == packetio.RTCPBufferPacket { + if reader := factory.GetRTCPReader(ssrc); reader != nil { + return reader + } + } + input := packetio.NewBuffer() + limit := 1_000_000 + if kind == packetio.RTCPBufferPacket { + limit = 100_000 + } + input.SetLimitSize(limit) + return input + } + registry := &interceptor.Registry{} + if err = webrtc.ConfigureRTCPReports(registry); err != nil { + return nil, err + } + publication.PC, err = webrtc.NewAPI(webrtc.WithMediaEngine(media), webrtc.WithInterceptorRegistry(registry), + webrtc.WithSettingEngine(settings)).NewPeerConnection(webrtc.Configuration{}) + if err != nil { + return nil, err + } + var transceiver *webrtc.RTPTransceiver + // Pion's diagnostic PC.ID is clock-based and can repeat. Source membership + // needs a unique physical owner even when signaling reuses a connection ID. + subscriberID := rand.Text() + for index := range options.Source.TrackInfo().Layers { + receiver := &publicationLayer{Source: options.Source, layer: int32(index), publication: publication} + track, trackErr := sfu.NewDownTrack(sfu.DownTrackParams{ + Codecs: []webrtc.RTPCodecParameters{codec}, Source: livekit.TrackSource_SCREEN_SHARE, + Receiver: receiver, BufferFactory: factory, + SubID: livekit.ParticipantID(subscriberID + "_" + strconv.Itoa(index)), + StreamID: options.Source.StreamID(), MaxTrack: options.Source.maxPackets, + Pacer: publication.pacer, Logger: log, Listener: publication, + RTCPWriter: publication.PC.WriteRTCP, DisableSenderReportPassThrough: true, + EnableStartAtDesiredQuality: true, + }) + if trackErr != nil { + return nil, trackErr + } + publication.tracks = append(publication.tracks, track) + track.SetMaxTemporalLayer(0) + track.SetMaxSpatialLayer(int32(index)) + local := &publicationTrack{DownTrack: track, id: options.Source.TrackID(), rid: PublicationRID(index, publication.activeCount), pacer: publication.pacer, counters: &publication.counters[index]} + if transceiver == nil { + transceiver, err = publication.PC.AddTransceiverFromTrack(local, webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendonly}) + } else { + err = transceiver.Sender().AddEncoding(local) + } + if err != nil { + return nil, err + } + local.transceiver = transceiver + track.SetTransceiver(transceiver) + publication.allocator.AddTrack(track, streamallocator.AddTrackParams{ + Source: livekit.TrackSource_SCREEN_SHARE, IsMultiLayered: true, + PublisherID: livekit.ParticipantID(options.Source.StreamID()), + }) + } + if options.Audio != nil { + sender, audioErr := publication.PC.AddTrack(&publicationAudio{TrackLocal: options.Audio, bytes: &publication.audioBytes}) + if audioErr != nil { + return nil, audioErr + } + publication.readers.Add(1) + go func() { + defer publication.readers.Done() + for { + if _, _, readErr := sender.ReadRTCP(); readErr != nil { + return + } + } + }() + } + return publication, nil +} + +func PublicationRID(layer, count int) string { + if count == 1 { + return "" + } + return []string{"q", "h", "f"}[layer] +} + +func (publication *Publication) SetConnected() error { + publication.mu.Lock() + defer publication.mu.Unlock() + if publication.closed { + return io.ErrClosedPipe + } + if publication.attached { + return nil + } + for _, track := range publication.tracks { + receiver := track.Receiver().(*publicationLayer) + receiver.attached = true + if err := receiver.AddDownTrack(track); err != nil { + receiver.attached = false + return err + } + track.SetConnected() + } + publication.attached = true + return nil +} + +func (publication *Publication) CurrentSource() *Source { + publication.mu.Lock() + defer publication.mu.Unlock() + return publication.source +} + +// ReplaceSource keeps every RID, allocator, counter and sender on this PC. +func (publication *Publication) ReplaceSource(next *Source) error { + recovery := false + // Source feedback may retire this publication; release all locks first. + defer func() { + if recovery { + for layer := range next.TrackInfo().Layers { + next.SendPLI(int32(layer), true) + } + } + }() + publication.mu.Lock() + defer publication.mu.Unlock() + if publication.closed { + return io.ErrClosedPipe + } + if next == nil { + return errors.New("replacement publication source is absent") + } + next.mu.Lock() + defer next.mu.Unlock() + if err := validateReplacement(publication.source, next); err != nil { + return err + } + if next == publication.source { + return nil + } + publication.writeMu.Lock() + defer publication.writeMu.Unlock() + for _, track := range publication.tracks { + if track.IsClosed() { + return io.ErrClosedPipe + } + } + for index, track := range publication.tracks { + receiver := &publicationLayer{Source: next, layer: int32(index), publication: publication, attached: publication.attached} + track.ReceiverRestart(track.Receiver()) + track.SetReceiver(receiver) + // The subscribed max layer did not change, so SetReceiver's layer + // notification alone does not wake the allocator for this fresh input. + publication.allocator.OnSubscriptionChanged(track) + } + publication.source = next + recovery = publication.attached + return nil +} + +func (publication *Publication) SetActiveCount(count int) error { + publication.mu.Lock() + defer publication.mu.Unlock() + if publication.closed { + return io.ErrClosedPipe + } + if count < 0 || count > len(publication.tracks) { + return errors.New("publication layer demand is invalid") + } + publication.activeCount = count + for index, track := range publication.tracks { + // Publisher demand is authoritative even while allocation is paused; + // subscription visibility mute may be ignored by the framework then. + track.PubMute(index >= count) + } + return nil +} + +func (publication *Publication) RequiredActiveCount() int { + publication.mu.Lock() + defer publication.mu.Unlock() + if publication.closed || !publication.attached { + return 0 + } + // The allocator still needs actual input for requested encodings when it + // probes recovery. This is codec demand, not affordable or delivered upload. + return publication.activeCount +} + +func (publication *Publication) SetAudioBitrate(bitrate uint32) { + publication.bandwidth.audio.Store(int64(bitrate)) +} +func (publication *Publication) VideoBudget() int64 { + return max(publication.bandwidth.target.Load()-publication.bandwidth.audio.Load(), 0) +} +func (publication *Publication) LowestLayerBudget() int64 { + remaining := publication.VideoBudget() + for _, track := range publication.tracks[1:] { + remaining -= track.BandwidthRequested() + } + return max(remaining, 0) +} + +type PublicationCounters struct { + Frames []uint64 + VideoBytes, AudioBytes uint64 + TargetBitrate int64 + Observed, Limited bool +} + +func (publication *Publication) Counters() PublicationCounters { + publication.mu.Lock() + defer publication.mu.Unlock() + stats := PublicationCounters{Frames: make([]uint64, len(publication.counters)), + AudioBytes: publication.audioBytes.Load(), TargetBitrate: publication.bandwidth.target.Load(), + Observed: publication.bandwidth.observed.Load(), Limited: publication.bandwidth.CongestionState() == bwe.CongestionStateCongested} + for index := range publication.counters { + stats.Frames[index] = publication.counters[index].frames.Load() + stats.VideoBytes += publication.counters[index].bytes.Load() + if index < publication.activeCount && publication.tracks[index].IsDeficient() { + stats.Limited = true + } + } + return stats +} + +func (publication *Publication) Close() error { + publication.closeOnce.Do(func() { + publication.mu.Lock() + publication.closed = true + publication.mu.Unlock() + if publication.PC != nil { + publication.closeErr = publication.PC.Close() + } + if publication.allocator != nil { + publication.allocator.Stop() + } + for _, track := range publication.tracks { + publication.source.DeleteDownTrack(track.SubscriberID()) + track.Close() + } + if publication.pacer != nil { + publication.pacer.Stop() + } + publication.readers.Wait() + }) + return publication.closeErr +} + +func (*Publication) OnBindAndConnected() {} +func (*Publication) OnStatsUpdate(*livekit.AnalyticsStat) {} +func (*Publication) OnMaxSubscribedLayerChanged(int32) {} +func (publication *Publication) OnRttUpdate(rtt uint32) { + publication.bandwidth.UpdateRTT(float64(rtt) / 1000) +} +func (*Publication) OnCodecNegotiated(webrtc.RTPCodecCapability) {} +func (*Publication) OnDownTrackClose(bool) {} +func (*Publication) OnStreamStarted(time.Duration) {} + +// Each RID exposes precisely one existing layer to the allocator; another RID +// can never substitute the same lower output and charge it twice. +type publicationLayer struct { + *Source + layer int32 + publication *Publication + attached bool +} + +func (layer *publicationLayer) AddDownTrack(sender sfu.TrackSender) error { + if !layer.attached { + return nil + } + filtered := &publicationLayerSender{ + DownTrack: sender.(*sfu.DownTrack), layer: layer.layer, receiver: layer, + } + filtered.registering.Store(true) + err := layer.Source.AddDownTrack(filtered) + filtered.registering.Store(false) + return err +} + +func (layer *publicationLayer) TrackID() livekit.TrackID { + return livekit.TrackID(string(layer.Source.TrackID()) + "_" + strconv.Itoa(int(layer.layer))) +} +func (layer *publicationLayer) GetLayeredBitrate() ([]int32, sfu.Bitrates) { + available, rates := layer.Source.GetLayeredBitrate() + return onlyPublicationLayer(layer.layer, available, rates) +} +func (*publicationLayer) SetMaxExpectedSpatialLayer(int32) {} + +func onlyPublicationLayer(layer int32, available []int32, rates sfu.Bitrates) ([]int32, sfu.Bitrates) { + var selected sfu.Bitrates + selected[layer] = rates[layer] + for _, index := range available { + if index == layer { + return []int32{layer}, selected + } + } + return nil, selected +} + +type publicationLayerSender struct { + *sfu.DownTrack + layer int32 + receiver *publicationLayer + registering atomic.Bool +} + +func (sender *publicationLayerSender) UpTrackBitrateReport(available []int32, rates sfu.Bitrates) { + sender.withReceiver(func() { + available, rates = onlyPublicationLayer(sender.layer, available, rates) + sender.DownTrack.UpTrackBitrateReport(available, rates) + }) +} +func (sender *publicationLayerSender) WriteRTP(packet *buffer.ExtPacket, layer int32) int32 { + sender.receiver.publication.writeMu.Lock() + defer sender.receiver.publication.writeMu.Unlock() + if sender.Receiver() != sender.receiver { + return 0 + } + if layer != sender.layer { + return 0 + } + return sender.DownTrack.WriteRTP(packet, layer) +} + +func (sender *publicationLayerSender) Close() { + sender.withReceiver(sender.DownTrack.Close) +} + +func (sender *publicationLayerSender) withReceiver(fn func()) { + if !sender.registering.Load() { + sender.receiver.publication.writeMu.Lock() + defer sender.receiver.publication.writeMu.Unlock() + } + if sender.Receiver() == sender.receiver { + fn() + } +} + +func (sender *publicationLayerSender) ReceiverRestart(receiver sfu.TrackReceiver) { + sender.withReceiver(func() { sender.DownTrack.ReceiverRestart(receiver) }) +} + +func (sender *publicationLayerSender) UpTrackLayersChange() { + sender.withReceiver(sender.DownTrack.UpTrackLayersChange) +} + +func (sender *publicationLayerSender) UpTrackBitrateAvailabilityChange() { + sender.withReceiver(sender.DownTrack.UpTrackBitrateAvailabilityChange) +} + +func (sender *publicationLayerSender) UpTrackMaxPublishedLayerChange(layer int32) { + sender.withReceiver(func() { sender.DownTrack.UpTrackMaxPublishedLayerChange(layer) }) +} + +func (sender *publicationLayerSender) UpTrackMaxTemporalLayerSeenChange(layer int32) { + sender.withReceiver(func() { sender.DownTrack.UpTrackMaxTemporalLayerSeenChange(layer) }) +} + +func (sender *publicationLayerSender) HandleRTCPSenderReportData(payloadType webrtc.PayloadType, layer int32, report *livekit.RTCPSenderReportState) error { + sender.receiver.publication.writeMu.Lock() + defer sender.receiver.publication.writeMu.Unlock() + if sender.Receiver() != sender.receiver { + return nil + } + return sender.DownTrack.HandleRTCPSenderReportData(payloadType, layer, report) +} + +type publicationTrack struct { + *sfu.DownTrack + id livekit.TrackID + rid string + transceiver *webrtc.RTPTransceiver + pacer *publicationPacer + counters *publicationCounters +} + +func (track *publicationTrack) ID() string { return string(track.id) } +func (track *publicationTrack) RID() string { return track.rid } +func (track *publicationTrack) Bind(context webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, error) { + var extension, midExtension uint8 + for _, header := range context.HeaderExtensions() { + if header.URI == publicationRIDExtension { + extension = uint8(header.ID) + } + if header.URI == "urn:ietf:params:rtp-hdrext:sdes:mid" { + midExtension = uint8(header.ID) + } + } + if track.rid != "" && extension == 0 { + return webrtc.RTPCodecParameters{}, errors.New("publication RID was not negotiated") + } + track.pacer.headersMu.Lock() + track.pacer.headers[uint32(context.SSRC())] = publicationHeaders{rid: track.rid, extension: extension, + mid: track.transceiver.Mid(), midExtension: midExtension, counters: track.counters} + track.pacer.headersMu.Unlock() + return track.DownTrack.Bind(context) +} + +type publicationHeaders struct { + rid, mid string + extension, midExtension uint8 + counters *publicationCounters +} + +type publicationCounters struct{ frames, bytes atomic.Uint64 } + +type publicationWriter struct { + webrtc.TrackLocalWriter + bytes *atomic.Uint64 + frames *atomic.Uint64 + primary bool +} + +func (writer *publicationWriter) WriteRTP(header *rtp.Header, payload []byte) (int, error) { + written, err := writer.TrackLocalWriter.WriteRTP(header, payload) + if err == nil && written > 0 { + writer.bytes.Add(uint64(header.MarshalSize() + len(payload))) + if writer.frames != nil && writer.primary && header.Marker && len(payload) > 0 { + writer.frames.Add(1) + } + } + return written, err +} + +type publicationAudio struct { + webrtc.TrackLocal + bytes *atomic.Uint64 +} + +func (audio *publicationAudio) Bind(context webrtc.TrackLocalContext) (webrtc.RTPCodecParameters, error) { + return audio.TrackLocal.Bind(publicationAudioContext{TrackLocalContext: context, + writer: &publicationWriter{TrackLocalWriter: context.WriteStream(), bytes: audio.bytes}}) +} + +type publicationAudioContext struct { + webrtc.TrackLocalContext + writer webrtc.TrackLocalWriter +} + +func (context publicationAudioContext) WriteStream() webrtc.TrackLocalWriter { return context.writer } + +// Opus uses its one existing encoder at an explicit fixed ceiling. Reserve its +// bytes from allocator capacities. TWCC observes video; reserving unobserved +// audio is conservative and does not claim that the estimator acknowledged it. +type publicationBWE struct { + *sendsidebwe.SendSideBWE + listener bwe.BWEListener + target atomic.Int64 + audio atomic.Int64 + observed atomic.Bool + pacer *publicationPacer + probeMu sync.Mutex + completedProbe ccutils.ProbeClusterInfo +} + +func (bandwidth *publicationBWE) HandleTWCCFeedback(report *rtcp.TransportLayerCC) { + bandwidth.observed.Store(true) + bandwidth.SendSideBWE.HandleTWCCFeedback(report) +} + +func (bandwidth *publicationBWE) SetBWEListener(listener bwe.BWEListener) { + bandwidth.listener = listener + bandwidth.SendSideBWE.SetBWEListener(bandwidth) +} +func (bandwidth *publicationBWE) OnCongestionStateChange(from, to bwe.CongestionState, capacity int64) { + if capacity > 0 { + bandwidth.target.Store(capacity) + bandwidth.pacer.SetBitrate(int(capacity)) + } + bandwidth.listener.OnCongestionStateChange(from, to, max(capacity-bandwidth.audio.Load(), 0)) +} +func (bandwidth *publicationBWE) ProbeClusterDone(info ccutils.ProbeClusterInfo) { + bandwidth.probeMu.Lock() + defer bandwidth.probeMu.Unlock() + bandwidth.completedProbe = info + bandwidth.SendSideBWE.ProbeClusterDone(info) +} +func (bandwidth *publicationBWE) ProbeClusterFinalize() (ccutils.ProbeSignal, int64, bool) { + bandwidth.probeMu.Lock() + defer bandwidth.probeMu.Unlock() + signal, capacity, done := bandwidth.SendSideBWE.ProbeClusterFinalize() + if done { + valid := sendsidebwe.DefaultSendSideBWEConfig.CongestionDetector.ProbeSignal.IsValid(bandwidth.completedProbe) + bandwidth.completedProbe = ccutils.ProbeClusterInfoInvalid + if !valid { + // StreamAllocator may upgrade on Inconclusive too. Its capacity must + // never promote the estimator's initial sentinel after an empty probe. + signal = ccutils.ProbeSignalInconclusive + capacity = min(capacity, bandwidth.target.Load()) + } else if signal == ccutils.ProbeSignalNotCongesting && capacity > bandwidth.target.Load() { + bandwidth.target.Store(capacity) + bandwidth.pacer.SetBitrate(int(capacity)) + } + } + return signal, max(capacity-bandwidth.audio.Load(), 0), done +} + +type publicationPacer struct { + *boundedPacer + bandwidth *publicationBWE + headersMu sync.Mutex + headers map[uint32]publicationHeaders +} + +func (queue *publicationPacer) Enqueue(packet *pacer.Packet) { + queue.headersMu.Lock() + headers := queue.headers[packet.Header.SSRC] + queue.headersMu.Unlock() + if headers.extension != 0 { + _ = packet.Header.SetExtension(headers.extension, []byte(headers.rid)) + } + if headers.midExtension != 0 { + _ = packet.Header.SetExtension(headers.midExtension, []byte(headers.mid)) + } + packet.HeaderSize = packet.Header.MarshalSize() + if headers.counters != nil { + packet.WriteStream = &publicationWriter{TrackLocalWriter: packet.WriteStream, + bytes: &headers.counters.bytes, frames: &headers.counters.frames, primary: !packet.IsRTX && !packet.IsProbe} + } + queue.boundedPacer.Enqueue(packet) +} +func (queue *publicationPacer) StartProbeCluster(info ccutils.ProbeClusterInfo) { + queue.boundedPacer.StartProbeCluster(info) + queue.SetBitrate(info.Goal.DesiredBps + int(queue.bandwidth.audio.Load())) +} +func (queue *publicationPacer) EndProbeCluster(id ccutils.ProbeClusterId) ccutils.ProbeClusterInfo { + info := queue.boundedPacer.EndProbeCluster(id) + queue.cancelProbes(id) + queue.SetBitrate(int(queue.bandwidth.target.Load())) + return info +} diff --git a/internal/media/forwarding/publication_test.go b/internal/media/forwarding/publication_test.go new file mode 100644 index 0000000..a1c0fbd --- /dev/null +++ b/internal/media/forwarding/publication_test.go @@ -0,0 +1,263 @@ +package forwarding + +import ( + "bytes" + "context" + "net" + "sync/atomic" + "testing" + "time" + + "github.com/TNTcraftHIM/Screener/internal/media/encoded" + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/bwe" + "github.com/livekit/livekit-server/pkg/sfu/bwe/sendsidebwe" + "github.com/livekit/livekit-server/pkg/sfu/ccutils" + "github.com/livekit/livekit-server/pkg/sfu/pacer" + "github.com/livekit/protocol/logger" + "github.com/pion/interceptor" + "github.com/pion/rtp/codecs" + "github.com/pion/webrtc/v4" + "github.com/pion/webrtc/v4/pkg/media" +) + +func TestPublicationEmptyProbeCannotIncreaseCapacity(t *testing.T) { + const target = int64(1_000_000) + const audio = int64(64_000) + log := logger.GetDiscardLogger() + bandwidth := &publicationBWE{SendSideBWE: sendsidebwe.NewSendSideBWE(sendsidebwe.SendSideBWEParams{ + Config: sendsidebwe.DefaultSendSideBWEConfig, Logger: log, + })} + bandwidth.target.Store(target) + bandwidth.audio.Store(audio) + bandwidth.pacer = &publicationPacer{bandwidth: bandwidth, boundedPacer: newBoundedPacer( + pacer.NewLeakyBucket(log, nil, 5*time.Millisecond, int(target)), log, 500, + )} + defer bandwidth.pacer.Stop() + info := ccutils.ProbeClusterInfo{Id: 1, CreatedAt: time.Now(), Goal: ccutils.ProbeClusterGoal{ + AvailableBandwidthBps: int(target), ExpectedUsageBps: int(target - audio), + DesiredBps: int(target * 2), Duration: bandwidth.ProbeDuration(), + }} + info.Goal.DesiredBytes = int(float64(info.Goal.DesiredBps) * info.Goal.Duration.Seconds() / 8) + bandwidth.ProbeClusterStarting(info) + bandwidth.pacer.StartProbeCluster(info) + time.Sleep(info.Goal.Duration) + info = bandwidth.pacer.EndProbeCluster(info.Id) + if info.Result.Bytes() != 0 || sendsidebwe.DefaultSendSideBWEConfig.CongestionDetector.ProbeSignal.IsValid(info) { + t.Fatal("fixture did not complete an empty invalid probe") + } + bandwidth.ProbeClusterDone(info) + deadline := time.Now().Add(sendsidebwe.DefaultSendSideBWEConfig.CongestionDetector.ProbePacketGroup.SettleWaitMax + time.Second) + for { + signal, capacity, done := bandwidth.ProbeClusterFinalize() + if done { + if signal != ccutils.ProbeSignalInconclusive || capacity > target-audio || bandwidth.target.Load() != target { + t.Fatalf("empty probe raised publication capacity: signal=%v capacity=%d target=%d", signal, capacity, bandwidth.target.Load()) + } + if bandwidth.completedProbe.Id != ccutils.ProbeClusterIdInvalid { + t.Fatal("completed probe was retained after finalization") + } + break + } + if time.Now().After(deadline) { + t.Fatal("empty probe did not finalize within the library settle bound") + } + time.Sleep(5 * time.Millisecond) + } +} + +func TestPublicationSimulcastReuseAudioDemandAndClose(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 8*time.Second) + defer cancel() + check := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + codec := webrtc.RTPCodecParameters{RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeVP8, ClockRate: 90_000, + RTCPFeedback: []webrtc.RTCPFeedback{{Type: "nack"}, {Type: "nack", Parameter: "pli"}, {Type: webrtc.TypeRTCPFBTransportCC}}, + }, PayloadType: 96} + source, err := NewEncodedSource(SourceOptions{ID: "screen", StreamID: "publication-fixture", Codec: codec, + Formats: []LayerFormat{{Width: 8, Height: 8, Bitrate: 90_000}, {Width: 8, Height: 8, Bitrate: 300_000}}}) + check(err) + defer source.Close() + settings := webrtc.SettingEngine{} + settings.SetNetworkTypes([]webrtc.NetworkType{webrtc.NetworkTypeUDP4}) + settings.SetIncludeLoopbackCandidate(true) + settings.SetIPFilter(func(ip net.IP) bool { return ip.IsLoopback() }) + audio, err := webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeOpus, ClockRate: 48_000, Channels: 2}, "audio", "publication-fixture") + check(err) + publication, err := NewPublication(TransportOptions{Source: source.Source, Settings: settings, ConnectionID: "publication", InitialBitrate: 1_000_000, Audio: audio}) + check(err) + defer publication.Close() + publication.SetAudioBitrate(64_000) + if publication.VideoBudget() != 936_000 { + t.Fatal("audio did not reserve the shared budget") + } + if len(publication.PC.GetSenders()) != 2 || len(publication.PC.GetSenders()[0].GetParameters().Encodings) != 2 { + t.Fatal("publication must use one simulcast video sender and one audio sender") + } + mediaEngine := &webrtc.MediaEngine{} + check(mediaEngine.RegisterDefaultCodecs()) + check(mediaEngine.RegisterHeaderExtension(webrtc.RTPHeaderExtensionCapability{URI: publicationRIDExtension}, webrtc.RTPCodecTypeVideo)) + check(mediaEngine.RegisterHeaderExtension(webrtc.RTPHeaderExtensionCapability{URI: "urn:ietf:params:rtp-hdrext:sdes:mid"}, webrtc.RTPCodecTypeVideo)) + registry := &interceptor.Registry{} + check(webrtc.RegisterDefaultInterceptors(mediaEngine, registry)) + check(webrtc.ConfigureTWCCSender(mediaEngine, registry)) + receiver, err := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine), webrtc.WithInterceptorRegistry(registry), webrtc.WithSettingEngine(settings)).NewPeerConnection(webrtc.Configuration{}) + check(err) + defer receiver.Close() + var received [3]atomic.Uint32 + var invalid atomic.Bool + receiver.OnTrack(func(track *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { + index := 2 + if track.Kind() == webrtc.RTPCodecTypeVideo { + if track.RID() == "q" { + index = 0 + } else if track.RID() == "h" { + index = 1 + } else { + invalid.Store(true) + return + } + } + for { + packet, _, readErr := track.ReadRTP() + if readErr != nil { + return + } + if len(packet.Payload) == 0 { + continue + } + if index < 2 { + var vp8 codecs.VP8Packet + payload, decodeErr := vp8.Unmarshal(packet.Payload) + if decodeErr != nil || !bytes.Equal(payload, sfu.VP8KeyFrame8x8) { + invalid.Store(true) + return + } + } + received[index].Add(1) + } + }) + offer, err := publication.PC.CreateOffer(nil) + check(err) + gather := webrtc.GatheringCompletePromise(publication.PC) + check(publication.PC.SetLocalDescription(offer)) + select { + case <-gather: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + check(receiver.SetRemoteDescription(*publication.PC.LocalDescription())) + answer, err := receiver.CreateAnswer(nil) + check(err) + gather = webrtc.GatheringCompletePromise(receiver) + check(receiver.SetLocalDescription(answer)) + select { + case <-gather: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + check(publication.PC.SetRemoteDescription(*receiver.LocalDescription())) + for publication.PC.ConnectionState() != webrtc.PeerConnectionStateConnected { + select { + case <-time.After(5 * time.Millisecond): + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + } + check(publication.SetConnected()) + check(publication.SetConnected()) + if len(source.GetDownTracks()) != 2 { + t.Fatal("publication did not attach exactly one output per encoding") + } + frame := 0 + write := func() { + frame++ + pts := time.Duration(frame) * time.Second / 30 + check(source.BeginFrame(pts, time.Now())) + for layer := 0; layer < 2; layer++ { + check(source.WriteFrame(layer, encoded.Frame{Data: sfu.VP8KeyFrame8x8, PTS: pts, Duration: time.Second / 30, Recovery: true})) + } + check(audio.WriteSample(media.Sample{Data: []byte{0xf8, 0xff, 0xfe}, Duration: 20 * time.Millisecond})) + } + for received[0].Load() < 3 || received[1].Load() < 3 || received[2].Load() < 3 { + write() + select { + case <-time.After(time.Second / 30): + case <-ctx.Done(): + t.Fatalf("publication RTP missing: low=%d high=%d audio=%d invalid=%t", received[0].Load(), received[1].Load(), received[2].Load(), invalid.Load()) + } + } + if invalid.Load() { + t.Fatal("publication changed encoded frame payloads") + } + if counters := publication.Counters(); counters.Frames[0] == 0 || counters.Frames[1] == 0 || counters.AudioBytes == 0 { + t.Fatalf("successful RTP writes were not accounted per encoding: %+v", counters) + } + // Once real bitrate measurements exist, one shared capacity update must + // constrain the sum of both RID allocations after reserving Opus. + for { + _, rates := source.GetLayeredBitrate() + if rates[0][0] > 0 && rates[1][0] > 0 { + break + } + write() + select { + case <-time.After(time.Second / 30): + case <-ctx.Done(): + t.Fatal("publication did not measure both available encodings") + } + } + publication.bandwidth.OnCongestionStateChange(bwe.CongestionStateNone, bwe.CongestionStateCongested, 65_000) + for publication.tracks[0].BandwidthRequested()+publication.tracks[1].BandwidthRequested() > 1_000 { + select { + case <-time.After(5 * time.Millisecond): + case <-ctx.Done(): + t.Fatal("simulcast allocations exceeded their shared post-audio capacity") + } + } + if publication.LowestLayerBudget() != 1_000 { + t.Fatal("paused higher output retained the lowest encoder's budget") + } + if publication.bandwidth.CongestionState() != bwe.CongestionStateNone || !publication.Counters().Limited { + t.Fatalf("clear transport feedback hid the committed publication allocation deficit: state=%v counters=%+v low=%v high=%v", publication.bandwidth.CongestionState(), publication.Counters(), publication.tracks[0].IsDeficient(), publication.tracks[1].IsDeficient()) + } + publication.bandwidth.OnCongestionStateChange(bwe.CongestionStateCongested, bwe.CongestionStateCongested, 1_000_000) + check(publication.SetActiveCount(1)) + if publication.RequiredActiveCount() != 1 { + t.Fatal("publication did not reduce its direct demand") + } + // Drain packets already admitted by the pacer before observing inactive output. + for range 6 { + write() + time.Sleep(time.Second / 30) + } + high, low := received[1].Load(), received[0].Load() + for range 6 { + write() + time.Sleep(time.Second / 30) + } + if received[1].Load() != high || received[0].Load() <= low { + t.Fatalf("inactive high RID affected low or continued sending: high=%d->%d low=%d->%d limited=%t lowtrack=%+v hightrack=%+v", high, received[1].Load(), low, received[0].Load(), publication.Counters().Limited, publication.tracks[0].DebugInfo(), publication.tracks[1].DebugInfo()) + } + if publication.Counters().Limited { + t.Fatal("publication stayed limited after requested low output recovered and high was deactivated") + } + check(publication.SetActiveCount(2)) + for received[1].Load() <= high { + write() + select { + case <-time.After(time.Second / 30): + case <-ctx.Done(): + t.Fatal("requested high RID did not resume after publisher activation") + } + } + check(publication.Close()) + if len(source.GetDownTracks()) != 0 { + t.Fatal("publication retained source outputs") + } +} diff --git a/internal/media/forwarding/recovery_test.go b/internal/media/forwarding/recovery_test.go new file mode 100644 index 0000000..d0d90f9 --- /dev/null +++ b/internal/media/forwarding/recovery_test.go @@ -0,0 +1,179 @@ +package forwarding + +import ( + "context" + "os" + "sync/atomic" + "testing" + "time" + + "github.com/TNTcraftHIM/Screener/internal/media/encoded" + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/pion/ice/v4" + "github.com/pion/interceptor" + "github.com/pion/logging" + "github.com/pion/transport/v4/vnet" + "github.com/pion/webrtc/v4" +) + +// Explicit CPU-only network gate: synthetic VP8-shaped payload, actual Pion +// ICE/DTLS/RTP/TWCC, and the production allocator/pacer/probe driver. No decoder, +// capture device or physical network participates. +func TestTransportConstrainedDormantRecovery(t *testing.T) { + if os.Getenv("SCREENER_TRANSPORT_RECOVERY") == "" { + t.Skip("set SCREENER_TRANSPORT_RECOVERY=1 for the bounded virtual-network gate") + } + ctx, cancel := context.WithTimeout(t.Context(), 32*time.Second) + defer cancel() + check := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + log := logging.NewDefaultLoggerFactory() + router, err := vnet.NewRouter(&vnet.RouterConfig{CIDR: "10.0.0.0/24", MinDelay: 20 * time.Millisecond, LoggerFactory: log}) + check(err) + sendNet, err := vnet.NewNet(&vnet.NetConfig{StaticIPs: []string{"10.0.0.1"}}) + check(err) + receiveNet, err := vnet.NewNet(&vnet.NetConfig{StaticIPs: []string{"10.0.0.2"}}) + check(err) + link := vnet.NewTBFQueue(6_000_000, 2400, 7500) + queue, err := vnet.NewQueue(receiveNet, link) + check(err) + defer queue.Close() + check(router.AddNet(sendNet)) + check(router.AddNet(queue)) + check(router.Start()) + defer router.Stop() + codec := webrtc.RTPCodecParameters{RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeVP8, ClockRate: 90_000, + RTCPFeedback: []webrtc.RTCPFeedback{{Type: "nack"}, {Type: "nack", Parameter: "pli"}, {Type: webrtc.TypeRTCPFBTransportCC}}, + }, PayloadType: 96} + source, err := NewEncodedSource(SourceOptions{ID: "recovery", StreamID: "recovery", Codec: codec, + Formats: []LayerFormat{{Width: 8, Height: 8, Bitrate: 300_000}, {Width: 16, Height: 16, Bitrate: 1_200_000}}, MaxPackets: 500}) + check(err) + defer source.Close() + sendSettings := webrtc.SettingEngine{} + sendSettings.SetNet(sendNet) + sendSettings.SetNetworkTypes([]webrtc.NetworkType{webrtc.NetworkTypeUDP4}) + sendSettings.SetICEMulticastDNSMode(ice.MulticastDNSModeDisabled) + transport, err := NewTransport(TransportOptions{Source: source.Source, Settings: sendSettings, + ConnectionID: "recovery-child", InitialBitrate: 1_200_000}) + check(err) + defer transport.Close() + receiveSettings := webrtc.SettingEngine{} + receiveSettings.SetNet(receiveNet) + receiveSettings.SetNetworkTypes([]webrtc.NetworkType{webrtc.NetworkTypeUDP4}) + receiveSettings.SetICEMulticastDNSMode(ice.MulticastDNSModeDisabled) + media := &webrtc.MediaEngine{} + check(media.RegisterDefaultCodecs()) + registry := &interceptor.Registry{} + check(webrtc.RegisterDefaultInterceptors(media, registry)) + check(webrtc.ConfigureTWCCSender(media, registry)) + receiver, err := webrtc.NewAPI(webrtc.WithMediaEngine(media), webrtc.WithInterceptorRegistry(registry), + webrtc.WithSettingEngine(receiveSettings)).NewPeerConnection(webrtc.Configuration{}) + check(err) + defer receiver.Close() + var received atomic.Uint64 + receiver.OnTrack(func(track *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { + for { + packet, _, readErr := track.ReadRTP() + if readErr != nil { + return + } + received.Add(uint64(len(packet.Payload))) + } + }) + offer, err := transport.PC.CreateOffer(nil) + check(err) + gathered := webrtc.GatheringCompletePromise(transport.PC) + check(transport.PC.SetLocalDescription(offer)) + select { + case <-gathered: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + check(receiver.SetRemoteDescription(*transport.PC.LocalDescription())) + answer, err := receiver.CreateAnswer(nil) + check(err) + gathered = webrtc.GatheringCompletePromise(receiver) + check(receiver.SetLocalDescription(answer)) + select { + case <-gathered: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + check(transport.PC.SetRemoteDescription(*receiver.LocalDescription())) + for transport.PC.ConnectionState() != webrtc.PeerConnectionStateConnected { + select { + case <-time.After(5 * time.Millisecond): + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + } + check(transport.SetConnected()) + period := time.Second / 30 + ticker := time.NewTicker(period) + defer ticker.Stop() + start, lastReport := time.Now(), time.Now() + var previousReceived uint64 + var dormant, recovered bool + minimum := int64(1_200_000) + for frame := 0; ; frame++ { + select { + case <-ticker.C: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + now := time.Now() + elapsed := now.Sub(start) + if elapsed >= 26*time.Second { + break + } + capacity := 6_000_000 + if elapsed >= 4*time.Second && elapsed < 12*time.Second { + capacity = 500_000 + } + link.SetRate(capacity) + state := transport.Output.State() + _, observed := transport.TargetBitrate() + high := !observed || state.Target == 1 || state.Prepare == 1 || !state.Paused && state.Current == 1 + if elapsed >= 7*time.Second && elapsed < 12*time.Second { + dormant = dormant || !high + minimum = min(minimum, state.VideoBudget) + } + pts := time.Duration(frame) * period + check(source.BeginFrame(pts, now)) + transport.Output.BeginFrame() + for layer, bitrate := range []int64{min(300_000, max(state.VideoBudget, 1)), 1_200_000} { + if layer == 1 && !high { + continue + } + data := make([]byte, max(len(sfu.VP8KeyFrame8x8), int(bitrate/8/30))) + copy(data, sfu.VP8KeyFrame8x8) + check(source.WriteFrame(layer, encoded.Frame{Data: data, PTS: pts, Duration: period})) + } + if now.Sub(lastReport) >= time.Second { + bytes := received.Load() + receivedBps := float64((bytes-previousReceived)*8) / now.Sub(lastReport).Seconds() + transport.mu.Lock() + t.Logf("%.1fs link=%d budget=%d target=%d current=%d high=%v receive=%.0f congestion=%v ready=%v probe=%d egress=%+v", + elapsed.Seconds(), capacity, state.VideoBudget, state.Target, state.Current, high, receivedBps, + transport.estimator.CongestionState(), transport.paddingReady, transport.probe.Id, transport.Egress()) + transport.mu.Unlock() + if elapsed > 15*time.Second && state.Current == 1 && receivedBps > 800_000 { + recovered = true + } + lastReport, previousReceived = now, bytes + } + } + if !dormant || minimum >= 1_200_000 || !recovered { + t.Fatalf("missing real constrained/dormant/released recovery: dormant=%v minimum=%d recovered=%v state=%+v", + dormant, minimum, recovered, transport.Output.State()) + } + check(transport.Close()) + if transport.prober.IsRunning() { + t.Fatal("closed transport retained a probe") + } +} diff --git a/internal/media/forwarding/retarget_test.go b/internal/media/forwarding/retarget_test.go new file mode 100644 index 0000000..f386f1a --- /dev/null +++ b/internal/media/forwarding/retarget_test.go @@ -0,0 +1,202 @@ +package forwarding + +import ( + "context" + "net" + "sync/atomic" + "testing" + "time" + + "github.com/TNTcraftHIM/Screener/internal/media/encoded" + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/pion/rtcp" + "github.com/pion/webrtc/v4" +) + +func TestSourceRetargetPreservesTransportAndPublication(t *testing.T) { + for _, simulcast := range []bool{false, true} { + name := "transport" + if simulcast { + name = "publication" + } + t.Run(name, func(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 8*time.Second) + defer cancel() + check := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + codec := webrtc.RTPCodecParameters{RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeVP8, ClockRate: 90_000, + RTCPFeedback: []webrtc.RTCPFeedback{{Type: "nack"}, {Type: "nack", Parameter: "pli"}}, + }, PayloadType: 96} + var keyframes atomic.Uint32 + makeSource := func(streamID string) *EncodedSource { + input, err := NewEncodedSource(SourceOptions{ID: "screen", StreamID: streamID, Codec: codec, + Formats: []LayerFormat{{Width: 8, Height: 8, Bitrate: 90_000}, {Width: 8, Height: 8, Bitrate: 300_000}}, + OnRTCP: func(_ int, packets []rtcp.Packet) { keyframes.Add(uint32(len(packets))) }, + }) + check(err) + t.Cleanup(input.Close) + return input + } + old, next := makeSource("retarget"), makeSource("retarget") + closed, foreign := makeSource("retarget"), makeSource("different-stream") + closed.Close() + settings := webrtc.SettingEngine{} + settings.SetNetworkTypes([]webrtc.NetworkType{webrtc.NetworkTypeUDP4}) + settings.SetIncludeLoopbackCandidate(true) + settings.SetIPFilter(func(ip net.IP) bool { return ip.IsLoopback() }) + options := TransportOptions{Source: old.Source, Settings: settings, ConnectionID: name, InitialBitrate: 1_000_000} + var pc *webrtc.PeerConnection + var replace func(*Source) error + var current func() *Source + var connect func() error + var closeTransport func() error + var transport *Transport + var publication *Publication + if simulcast { + var err error + publication, err = NewPublication(options) + check(err) + pc, replace, current, connect, closeTransport = publication.PC, publication.ReplaceSource, + publication.CurrentSource, publication.SetConnected, publication.Close + } else { + var err error + transport, err = NewTransport(options) + check(err) + pc, replace, current, connect, closeTransport = transport.PC, transport.ReplaceSource, + transport.CurrentSource, transport.SetConnected, transport.Close + } + defer closeTransport() + check(replace(next.Source)) + if len(next.GetDownTracks()) != 0 { + t.Fatal("retarget attached an unconnected consumer") + } + check(replace(old.Source)) + mediaEngine := &webrtc.MediaEngine{} + check(mediaEngine.RegisterDefaultCodecs()) + if simulcast { + check(mediaEngine.RegisterHeaderExtension(webrtc.RTPHeaderExtensionCapability{URI: publicationRIDExtension}, webrtc.RTPCodecTypeVideo)) + check(mediaEngine.RegisterHeaderExtension(webrtc.RTPHeaderExtensionCapability{URI: "urn:ietf:params:rtp-hdrext:sdes:mid"}, webrtc.RTPCodecTypeVideo)) + } + receiver, err := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine), webrtc.WithSettingEngine(settings)).NewPeerConnection(webrtc.Configuration{}) + check(err) + defer receiver.Close() + var received [2]atomic.Uint32 + var badTimeline atomic.Bool + receiver.OnTrack(func(track *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { + index := 0 + if track.RID() == "h" { + index = 1 + } + var previous uint32 + for { + packet, _, err := track.ReadRTP() + if err != nil { + return + } + if packet.Marker && len(packet.Payload) > 0 { + if previous != 0 && int32(packet.Timestamp-previous) <= 0 { + badTimeline.Store(true) + t.Logf("RID=%s RTP timestamp=%d previous=%d seq=%d", track.RID(), packet.Timestamp, previous, packet.SequenceNumber) + } + previous = packet.Timestamp + received[index].Add(1) + } + } + }) + offer, err := pc.CreateOffer(nil) + check(err) + gather := webrtc.GatheringCompletePromise(pc) + check(pc.SetLocalDescription(offer)) + select { + case <-gather: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + check(receiver.SetRemoteDescription(*pc.LocalDescription())) + answer, err := receiver.CreateAnswer(nil) + check(err) + gather = webrtc.GatheringCompletePromise(receiver) + check(receiver.SetLocalDescription(answer)) + select { + case <-gather: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + check(pc.SetRemoteDescription(*receiver.LocalDescription())) + for pc.ConnectionState() != webrtc.PeerConnectionStateConnected { + select { + case <-time.After(5 * time.Millisecond): + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + } + check(connect()) + frame := 0 + write := func(input *EncodedSource) { + frame++ + pts := time.Duration(frame) * time.Second / 30 + check(input.BeginFrame(pts, time.Now())) + for layer := 0; layer < 2; layer++ { + check(input.WriteFrame(layer, encoded.Frame{Data: sfu.VP8KeyFrame8x8, PTS: pts, Duration: time.Second / 30, Recovery: true})) + } + } + await := func(input *EncodedSource, first, second uint32) { + for received[0].Load() < first || simulcast && received[1].Load() < second { + write(input) + select { + case <-time.After(time.Second / 30): + case <-ctx.Done(): + t.Fatalf("retarget media stalled: first=%d second=%d", received[0].Load(), received[1].Load()) + } + } + } + await(old, 3, 3) + for _, rejected := range []*Source{nil, closed.Source, foreign.Source} { + if replace(rejected) == nil || current() != old.Source { + t.Fatal("invalid replacement changed the active source") + } + } + oldSenders := old.GetDownTracks() + first, second := received[0].Load(), received[1].Load() + beforePLI := keyframes.Load() + write(next) + check(replace(next.Source)) + if current() != next.Source || len(old.GetDownTracks()) != 0 || len(next.GetDownTracks()) != len(oldSenders) { + t.Fatal("receiver membership did not move once") + } + for _, stale := range oldSenders { + if stale.WriteRTP(nil, 0) != 0 { + t.Fatal("detached source forwarded a late packet") + } + stale.Close() + stale.ReceiverRestart(old.Source) + check(stale.HandleRTCPSenderReportData(codec.PayloadType, 0, nil)) + stale.UpTrackMaxPublishedLayerChange(-1) + stale.UpTrackMaxTemporalLayerSeenChange(-1) + } + old.Close() + await(next, first+6, second+6) + if pc.ConnectionState() != webrtc.PeerConnectionStateConnected || badTimeline.Load() || keyframes.Load() <= beforePLI { + t.Fatalf("retarget lost connection, timeline or recovery: state=%v timeline=%v PLI=%d->%d", pc.ConnectionState(), badTimeline.Load(), beforePLI, keyframes.Load()) + } + if transport != nil && (transport.PC != pc || transport.Output.State().Current < 0 || transport.Egress().Packets == 0) { + t.Fatal("retarget bypassed the output wrapper or counters") + } + if publication != nil { + counters := publication.Counters() + if publication.PC != pc || counters.Frames[0] < 6 || counters.Frames[1] < 6 { + t.Fatal("retarget lost per-RID forwarding or counters") + } + } + check(closeTransport()) + if len(next.GetDownTracks()) != 0 || replace(old.Source) == nil { + t.Fatal("retarget retained membership after closure") + } + }) + } +} diff --git a/internal/media/forwarding/source.go b/internal/media/forwarding/source.go new file mode 100644 index 0000000..bcd2039 --- /dev/null +++ b/internal/media/forwarding/source.go @@ -0,0 +1,308 @@ +package forwarding + +import ( + "cmp" + "errors" + "slices" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/TNTcraftHIM/Screener/internal/media/encoded" + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/mediatransportutil" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/livekit/protocol/utils" + "github.com/livekit/protocol/utils/mono" + "github.com/pion/rtcp" + "github.com/pion/rtp" + "github.com/pion/webrtc/v4" +) + +type LayerFormat struct { + Width, Height, Bitrate uint32 +} + +type SourceOptions struct { + ID, StreamID string + Codec webrtc.RTPCodecParameters + Formats []LayerFormat + HeaderExtensions []webrtc.RTPHeaderExtensionParameter + MaxPackets int + OnRTCP func(layer int, packets []rtcp.Packet) +} + +// Source owns finite RTP layers and their mature forwarding buffers. The caller +// owns input generations, codec clocks and PeerConnections. Feedback callbacks +// run without Source's mutex and may retire the source. +type Source struct { + *sfu.ReceiverBase + mu sync.Mutex + buffers []*buffer.Buffer + maxPackets int + onRTCP func(int, []rtcp.Packet) + closed bool + currentTrackInfo atomic.Pointer[livekit.TrackInfo] + lowestRateControlled atomic.Bool +} + +// The live codec owner sets this after startup and clears it at retirement. +func (source *Source) SetLowestLayerRateControlled(controlled bool) { + source.lowestRateControlled.Store(controlled) +} + +func (source *Source) RateControlled(layer int32) bool { + return layer == 0 && !source.IsClosed() && source.lowestRateControlled.Load() +} + +func NewSource(options SourceOptions) (*Source, error) { + if options.ID == "" || options.StreamID == "" || + options.Codec.ClockRate != 90_000 || + (!strings.EqualFold(options.Codec.MimeType, webrtc.MimeTypeH264) && + !strings.EqualFold(options.Codec.MimeType, webrtc.MimeTypeVP8)) { + return nil, errors.New("forwarded video source is invalid") + } + if options.MaxPackets == 0 { + options.MaxPackets = encoded.MaxPacketWindow + } + if options.MaxPackets < buffer.InitPacketBufferSizeVideo || options.MaxPackets > 1<<16 { + return nil, errors.New("forwarded video packet bound is invalid") + } + info := &livekit.TrackInfo{ + Sid: options.ID, Type: livekit.TrackType_VIDEO, Source: livekit.TrackSource_SCREEN_SHARE, + MimeType: options.Codec.MimeType, + } + if err := setLayerFormats(info, options.Formats); err != nil { + return nil, err + } + source := &Source{ + ReceiverBase: sfu.NewReceiverBase(sfu.ReceiverBaseParams{ + TrackID: livekit.TrackID(options.ID), StreamID: options.StreamID, + Kind: webrtc.RTPCodecTypeVideo, Codec: options.Codec, + HeaderExtensions: canonicalHeaderExtensions(options.HeaderExtensions), Logger: logger.GetDiscardLogger(), + StreamTrackerManagerConfig: sfu.DefaultStreamTrackerManagerConfig, + }, info, sfu.ReceiverCodecStateNormal), + buffers: make([]*buffer.Buffer, len(options.Formats)), + maxPackets: options.MaxPackets, onRTCP: options.OnRTCP, + } + source.currentTrackInfo.Store(info) + return source, nil +} + +func (source *Source) TrackInfo() *livekit.TrackInfo { + return utils.CloneProto(source.currentTrackInfo.Load()) +} + +func canonicalHeaderExtensions(extensions []webrtc.RTPHeaderExtensionParameter) []webrtc.RTPHeaderExtensionParameter { + result := slices.Clone(extensions) + slices.SortFunc(result, func(left, right webrtc.RTPHeaderExtensionParameter) int { + if order := cmp.Compare(left.ID, right.ID); order != 0 { + return order + } + return cmp.Compare(left.URI, right.URI) + }) + return result +} + +func setLayerFormats(info *livekit.TrackInfo, formats []LayerFormat) error { + if len(formats) < 1 || len(formats) > int(buffer.DefaultMaxLayerSpatial)+1 { + return errors.New("forwarded video layer count is invalid") + } + layers := make([]*livekit.VideoLayer, len(formats)) + for index, format := range formats { + if (format.Width == 0) != (format.Height == 0) { + return errors.New("forwarded video layer format is invalid") + } + quality := livekit.VideoQuality(index) + if index == len(formats)-1 { + quality = livekit.VideoQuality_HIGH + } + layers[index] = &livekit.VideoLayer{ + Quality: quality, SpatialLayer: int32(index), Rid: buffer.DefaultVideoLayersRid[index], + Width: format.Width, Height: format.Height, Bitrate: format.Bitrate, + } + } + highest := formats[len(formats)-1] + info.Width, info.Height = highest.Width, highest.Height + info.Simulcast, info.Layers = len(layers) > 1, layers + info.Codecs = []*livekit.SimulcastCodecInfo{{ + MimeType: info.MimeType, VideoLayerMode: livekit.VideoLayer_ONE_SPATIAL_LAYER_PER_STREAM, Layers: layers, + }} + return nil +} + +// UpdateFormats changes one source owner's declared layer metadata without +// replacing its buffers or packet identities. Zero values remain unknown. +// The caller serializes updates with its capture/source generation changes. +func (source *Source) UpdateFormats(formats []LayerFormat) error { + source.mu.Lock() + closed, count := source.closed, len(source.buffers) + source.mu.Unlock() + if closed { + return sfu.ErrReceiverClosed + } + if len(formats) != count { + return errors.New("forwarded video layer count changed") + } + info := source.TrackInfo() + if err := setLayerFormats(info, formats); err != nil { + return err + } + source.currentTrackInfo.Store(info) + // The library can notify downtracks while applying metadata. + source.UpdateTrackInfo(info) + return nil +} + +// UpdateDimensions publishes actual encoder dimensions without UpdateTrackInfo's +// mute propagation, which resets every active layer's tracker and bitrate. +func (source *Source) UpdateDimensions(layer int, width, height uint32) error { + source.mu.Lock() + defer source.mu.Unlock() + if source.closed { + return sfu.ErrReceiverClosed + } + if layer < 0 || layer >= len(source.buffers) || width == 0 || height == 0 { + return errors.New("forwarded video layer dimensions are invalid") + } + current := source.currentTrackInfo.Load() + if current.Layers[layer].Width == width && current.Layers[layer].Height == height { + return nil + } + info := utils.CloneProto(current) + info.Layers[layer].Width, info.Layers[layer].Height = width, height + info.Codecs[0].Layers[layer].Width, info.Codecs[0].Layers[layer].Height = width, height + if layer == len(source.buffers)-1 { + info.Width, info.Height = width, height + } + source.currentTrackInfo.Store(info) + return nil +} + +// BindLayer installs a single source identity. A different SSRC requires the +// caller's source-generation replacement, not silent buffer reuse. +func (source *Source) BindLayer(layer int, ssrc uint32, parameters webrtc.RTPParameters) error { + source.mu.Lock() + defer source.mu.Unlock() + if source.closed { + return sfu.ErrReceiverClosed + } + if layer < 0 || layer >= len(source.buffers) || ssrc == 0 { + return sfu.ErrInvalidLayer + } + if source.buffers[layer] != nil { + return sfu.ErrDuplicateLayer + } + codec := source.Codec() + if !slices.ContainsFunc(parameters.Codecs, func(candidate webrtc.RTPCodecParameters) bool { + return candidate.PayloadType == codec.PayloadType && candidate.ClockRate == codec.ClockRate && + strings.EqualFold(candidate.MimeType, codec.MimeType) && candidate.SDPFmtpLine == codec.SDPFmtpLine + }) || !slices.Equal(canonicalHeaderExtensions(parameters.HeaderExtensions), source.HeaderExtensions()) { + return errors.New("forwarded video layer codec does not match the source") + } + for _, existing := range source.buffers { + if existing != nil && existing.SSRC() == ssrc { + return errors.New("forwarded video layers cannot share an SSRC") + } + } + input := buffer.NewBuffer(ssrc, source.maxPackets, buffer.InitPacketBufferSizeAudio) + input.SetLogger(logger.GetDiscardLogger()) + if err := input.Bind(parameters, codec.RTPCodecCapability, int(source.TrackInfo().Layers[layer].Bitrate)); err != nil { + _ = input.Close() + return err + } + if source.onRTCP != nil { + input.OnRtcpFeedback(func(packets []rtcp.Packet) { source.onRTCP(layer, packets) }) + } + source.AddBuffer(input, int32(layer)) + source.buffers[layer] = input + source.StartBuffer(input, int32(layer)) + return nil +} + +func (source *Source) layerBuffer(layer int) (*buffer.Buffer, error) { + source.mu.Lock() + defer source.mu.Unlock() + if source.closed { + return nil, sfu.ErrReceiverClosed + } + if layer < 0 || layer >= len(source.buffers) || source.buffers[layer] == nil { + return nil, sfu.ErrInvalidLayer + } + return source.buffers[layer], nil +} + +func (source *Source) WriteRTP(layer int, packet *rtp.Packet) error { + input, err := source.layerBuffer(layer) + if err != nil { + return err + } + if packet == nil || packet.Version != 2 || packet.SSRC != input.SSRC() || + packet.PayloadType != uint8(source.Codec().PayloadType) { + return errors.New("forwarded RTP packet does not match its layer") + } + data, err := packet.Marshal() + if err != nil { + return err + } + _, err = input.Write(data) + return err +} + +func (source *Source) SenderReport(layer int, report *rtcp.SenderReport) error { + input, err := source.layerBuffer(layer) + if err != nil { + return err + } + if report == nil || report.SSRC != input.SSRC() { + return errors.New("forwarded sender report does not match its layer") + } + input.SetSenderReportData(&livekit.RTCPSenderReportState{ + RtpTimestamp: report.RTPTime, NtpTimestamp: report.NTPTime, + Packets: report.PacketCount, Octets: uint64(report.OctetCount), At: mono.UnixNano(), + }) + return nil +} + +// ReferenceTime keeps derived outputs on the received source's RTP/NTP clock. +// The bounded input history makes the nearest 32-bit timestamp delta unambiguous. +func (source *Source) ReferenceTime(layer int, timestamp uint32) (time.Time, bool) { + input, err := source.layerBuffer(layer) + if err != nil { + return time.Time{}, false + } + report := input.GetSenderReportData() + if report == nil || report.NtpTimestamp == 0 { + return time.Time{}, false + } + delta := time.Duration(int32(timestamp-report.RtpTimestamp)) * time.Second / time.Duration(source.Codec().ClockRate) + return mediatransportutil.NtpTime(report.NtpTimestamp).Time().Add(delta), true +} + +// SetCorrelation receives the local source owner's RTP/NTP mapping unchanged. +// The layer must already have received RTP; the media core owns report ordering. +func (source *Source) SetCorrelation(layer int, correlation *livekit.RTCPSenderReportState) error { + input, err := source.layerBuffer(layer) + if err != nil { + return err + } + if correlation == nil { + return errors.New("forwarded source correlation is absent") + } + input.SetSenderReportData(correlation) + return nil +} + +func (source *Source) Close() { + source.mu.Lock() + if source.closed { + source.mu.Unlock() + return + } + source.closed = true + source.mu.Unlock() + source.ReceiverBase.Close("source closed", true) +} diff --git a/internal/media/forwarding/source_test.go b/internal/media/forwarding/source_test.go new file mode 100644 index 0000000..e7a2400 --- /dev/null +++ b/internal/media/forwarding/source_test.go @@ -0,0 +1,121 @@ +package forwarding + +import ( + "errors" + "testing" + "time" + + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/mediatransportutil" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/utils/mono" + "github.com/pion/rtcp" + "github.com/pion/rtp" + "github.com/pion/webrtc/v4" +) + +func TestSourceLayerBindingFeedbackAndClose(t *testing.T) { + codec := webrtc.RTPCodecParameters{RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeVP8, ClockRate: 90_000, + RTCPFeedback: []webrtc.RTCPFeedback{{Type: "nack", Parameter: "pli"}}, + }, PayloadType: 96} + var source *Source + feedbackLayer := -1 + var err error + source, err = NewSource(SourceOptions{ + ID: "source", StreamID: "screen", Codec: codec, + Formats: []LayerFormat{{}, {Width: 640, Height: 360, Bitrate: 1_200_000}}, + HeaderExtensions: []webrtc.RTPHeaderExtensionParameter{ + {ID: 1, URI: "urn:ietf:params:rtp-hdrext:sdes:mid"}, + {ID: 2, URI: "urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id"}, + }, + OnRTCP: func(layer int, packets []rtcp.Packet) { + for _, packet := range packets { + if _, ok := packet.(*rtcp.PictureLossIndication); ok { + feedbackLayer = layer + source.Close() + } + } + }, + }) + if err != nil { + t.Fatal(err) + } + t.Cleanup(source.Close) + source.SetLowestLayerRateControlled(true) + if !source.RateControlled(0) || source.RateControlled(1) { + t.Fatal("local codec ownership escaped its lowest output") + } + if info := source.TrackInfo(); info.Layers[0].Bitrate != 0 || info.Layers[0].Width != 0 { + t.Fatal("unknown received metadata was replaced by a guessed value") + } + parameters := webrtc.RTPParameters{Codecs: []webrtc.RTPCodecParameters{codec}, + HeaderExtensions: []webrtc.RTPHeaderExtensionParameter{ + {ID: 2, URI: "urn:ietf:params:rtp-hdrext:sdes:rtp-stream-id"}, + {ID: 1, URI: "urn:ietf:params:rtp-hdrext:sdes:mid"}, + }} + if source.BindLayer(2, 1000, parameters) == nil || source.BindLayer(0, 1000, webrtc.RTPParameters{}) == nil { + t.Fatal("invalid layer or codec parameters were accepted") + } + if err = source.BindLayer(0, 1000, parameters); err != nil { + t.Fatal(err) + } + if !errors.Is(source.BindLayer(0, 1001, parameters), sfu.ErrDuplicateLayer) || + source.BindLayer(1, 1000, parameters) == nil { + t.Fatal("duplicate source identity was accepted") + } + if err = source.BindLayer(1, 1001, parameters); err != nil { + t.Fatal(err) + } + initialBuffer := source.GetAllBuffers()[0] + if source.UpdateFormats([]LayerFormat{{Width: 320, Height: 180, Bitrate: 300_000}}) == nil { + t.Fatal("live metadata changed the fixed layer count") + } + if err = source.UpdateFormats([]LayerFormat{ + {Width: 320, Height: 180, Bitrate: 300_000}, + {Width: 1280, Height: 720, Bitrate: 3_000_000}, + }); err != nil { + t.Fatal(err) + } + if info := source.TrackInfo(); info.Width != 1280 || info.Layers[0].Bitrate != 300_000 || + source.GetAllBuffers()[0] != initialBuffer || source.Codec().PayloadType != codec.PayloadType { + t.Fatal("live metadata did not preserve source buffer and codec ownership") + } + packet := &rtp.Packet{Header: rtp.Header{ + Version: 2, SSRC: 1000, PayloadType: 96, SequenceNumber: 100, Timestamp: 9000, Marker: true, + }, Payload: []byte{0x10, 0, 0, 0, 0x9d, 0x01, 0x2a, 0xa0, 0, 0x5a, 0}} + if source.WriteRTP(1, packet) == nil { + t.Fatal("RTP crossed its layer identity") + } + if err = source.WriteRTP(0, packet); err != nil { + t.Fatal(err) + } + report := &rtcp.SenderReport{SSRC: 1000, RTPTime: 9000, + NTPTime: uint64(mediatransportutil.ToNtpTime(time.Now())), PacketCount: 1, OctetCount: 11} + if err = source.SenderReport(0, report); err != nil { + t.Fatal(err) + } + input := source.GetAllBuffers()[0] + if stored := input.GetSenderReportData(); stored == nil || stored.NtpTimestamp != report.NTPTime || stored.RtpTimestamp != 9000 { + t.Fatalf("original sender clock was changed: %+v", stored) + } + if err = source.SetCorrelation(0, &livekit.RTCPSenderReportState{ + RtpTimestamp: 99000, NtpTimestamp: report.NTPTime + 1<<32, + Packets: 2, Octets: 22, At: mono.UnixNano(), + }); err != nil { + t.Fatal(err) + } + source.SendPLI(0, true) + if feedbackLayer != 0 || !source.IsClosed() || !errors.Is(source.WriteRTP(0, packet), sfu.ErrReceiverClosed) { + t.Fatal("feedback could not retire its source without retaining stale writes") + } + if source.RateControlled(0) { + t.Fatal("closed source retained local codec ownership") + } + if err = source.BindLayer(0, 2000, parameters); !errors.Is(err, sfu.ErrReceiverClosed) { + t.Fatal("closed source accepted another layer") + } + if err = source.UpdateFormats([]LayerFormat{{}, {}}); !errors.Is(err, sfu.ErrReceiverClosed) { + t.Fatal("closed source accepted new metadata") + } +} diff --git a/internal/media/forwarding/transport.go b/internal/media/forwarding/transport.go new file mode 100644 index 0000000..e78f64f --- /dev/null +++ b/internal/media/forwarding/transport.go @@ -0,0 +1,635 @@ +package forwarding + +import ( + "errors" + "io" + "slices" + "sync" + "sync/atomic" + "time" + + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/buffer" + "github.com/livekit/livekit-server/pkg/sfu/bwe" + "github.com/livekit/livekit-server/pkg/sfu/bwe/sendsidebwe" + "github.com/livekit/livekit-server/pkg/sfu/ccutils" + "github.com/livekit/livekit-server/pkg/sfu/pacer" + "github.com/livekit/livekit-server/pkg/sfu/streamallocator" + "github.com/livekit/protocol/livekit" + "github.com/livekit/protocol/logger" + "github.com/pion/interceptor" + "github.com/pion/rtcp" + "github.com/pion/rtp" + "github.com/pion/transport/v4/packetio" + "github.com/pion/webrtc/v4" +) + +type TransportOptions struct { + Source *Source + Settings webrtc.SettingEngine + ConnectionID string + InitialBitrate int + Audio webrtc.TrackLocal + // OnDemandChanged must only signal its owner without blocking or reentering + // Transport. Callers can coalesce signals in their existing bounded owner. + OnDemandChanged func() +} + +// Transport is outgoing-only. Its caller owns signaling, ICE events and the +// shared UDP mux. Video RTCP is callback-driven; never read VideoSender's RTCP. +type Transport struct { + PC *webrtc.PeerConnection + Output *Output + VideoSender *webrtc.RTPSender + AudioSender *webrtc.RTPSender + source *Source + estimator *sendsidebwe.SendSideBWE + prober *ccutils.Prober + probe ccutils.ProbeClusterInfo + target atomic.Int64 + pacer *boundedPacer + observed atomic.Bool + mu sync.Mutex + attached bool + closed bool + audioBitrate uint32 + paddingReady bool + closeOnce sync.Once + closeErr error + readers sync.WaitGroup +} + +func NewTransport(options TransportOptions) (_ *Transport, err error) { + if options.Source == nil || options.ConnectionID == "" || options.InitialBitrate <= 0 { + return nil, errors.New("forwarding transport options are invalid") + } + transport := &Transport{source: options.Source} + transport.target.Store(int64(options.InitialBitrate)) + defer func() { + if err != nil { + _ = transport.Close() + } + }() + codec := options.Source.Codec() + media := &webrtc.MediaEngine{} + if err = media.RegisterCodec(codec, webrtc.RTPCodecTypeVideo); err != nil { + return nil, err + } + if options.Audio != nil { + audio, ok := options.Audio.(interface { + Codec() webrtc.RTPCodecCapability + }) + if !ok { + return nil, errors.New("forwarding audio codec is unavailable") + } + if err = media.RegisterCodec(webrtc.RTPCodecParameters{ + RTPCodecCapability: audio.Codec(), PayloadType: 111, + }, webrtc.RTPCodecTypeAudio); err != nil { + return nil, err + } + } + factory := buffer.NewFactoryOfBufferFactory(options.Source.maxPackets, buffer.InitPacketBufferSizeAudio).CreateBufferFactory() + settings := options.Settings + settings.BufferFactory = func(kind packetio.BufferPacketType, ssrc uint32) io.ReadWriteCloser { + if kind == packetio.RTCPBufferPacket { + if reader := factory.GetRTCPReader(ssrc); reader != nil { + return reader + } + } + // Preserve stock Pion SRTP/SRTCP bounds for unclaimed streams, including audio. + input := packetio.NewBuffer() + limit := 1_000_000 + if kind == packetio.RTCPBufferPacket { + limit = 100_000 + } + input.SetLimitSize(limit) + return input + } + registry := &interceptor.Registry{} + // LiveKit records actual paced sends and owns this extension's sequence space. + if err = media.RegisterHeaderExtension(webrtc.RTPHeaderExtensionCapability{ + URI: "http://www.ietf.org/id/draft-holmer-rmcat-transport-wide-cc-extensions-01", + }, webrtc.RTPCodecTypeVideo); err != nil { + return nil, err + } + // DownTrack owns video retransmission; registering default NACK interceptors + // here would create a second responder. Pion still owns normal RTCP reports. + if err = webrtc.ConfigureRTCPReports(registry); err != nil { + return nil, err + } + transport.PC, err = webrtc.NewAPI(webrtc.WithMediaEngine(media), + webrtc.WithInterceptorRegistry(registry), webrtc.WithSettingEngine(settings)).NewPeerConnection(webrtc.Configuration{}) + if err != nil { + return nil, err + } + log := logger.GetDiscardLogger() + transport.estimator = sendsidebwe.NewSendSideBWE(sendsidebwe.SendSideBWEParams{ + Config: sendsidebwe.DefaultSendSideBWEConfig, Logger: log, + }) + transport.estimator.SetBWEListener(transport) + transport.prober = ccutils.NewProber(ccutils.ProberParams{Listener: transport, Logger: log}) + // Use the pinned Pion pacer's 5 ms service cadence, with LiveKit packet + // ownership and a cap matching this connection's retransmission cache. + transport.pacer = newBoundedPacer(pacer.NewLeakyBucket(log, nil, 5*time.Millisecond, options.InitialBitrate), + log, options.Source.maxPackets) + transport.pacer.SetBitrate(options.InitialBitrate) + transport.pacer.sender = pacer.NewBase(log, transport.estimator) + transport.pacer.SetPacerProbeObserverListener(transport) + track, err := sfu.NewDownTrack(sfu.DownTrackParams{ + Codecs: []webrtc.RTPCodecParameters{codec}, Source: livekit.TrackSource_SCREEN_SHARE, + Receiver: &outputReceiver{Source: options.Source}, BufferFactory: factory, SubID: livekit.ParticipantID(options.ConnectionID), + StreamID: options.Source.StreamID(), MaxTrack: options.Source.maxPackets, + Pacer: transport.pacer, Logger: log, Listener: transport, + RTCPWriter: transport.PC.WriteRTCP, DisableSenderReportPassThrough: true, + EnableStartAtDesiredQuality: true, + }) + if err != nil { + return nil, err + } + transport.Output = NewOutput(track, int64(options.InitialBitrate)) + track.Receiver().(*outputReceiver).output = transport.Output + transport.Output.onActivity = transport.mediaActivity + transport.Output.onDemandChanged = options.OnDemandChanged + track.SetStreamAllocatorListener(transport) + track.AddReceiverReportListener(func(*sfu.DownTrack, *rtcp.ReceiverReport) { + transport.mu.Lock() + defer transport.mu.Unlock() + if !transport.closed { + transport.paddingReady = true + transport.driveProbeLocked() + } + }) + // Spatial selection arms acquisition; its temporal bound must already be valid. + transport.Output.SetMaxTemporalLayer(0) + transport.Output.SetMaxSpatialLayer(int32(len(options.Source.TrackInfo().Layers) - 1)) + transceiver, err := transport.PC.AddTransceiverFromTrack(track, + webrtc.RTPTransceiverInit{Direction: webrtc.RTPTransceiverDirectionSendonly}) + if err != nil { + return nil, err + } + track.SetTransceiver(transceiver) + transport.VideoSender = transceiver.Sender() + if options.Audio != nil { + transport.AudioSender, err = transport.PC.AddTrack(options.Audio) + if err != nil { + return nil, err + } + transport.readers.Add(1) + go func() { + defer transport.readers.Done() + for { + if _, _, readErr := transport.AudioSender.ReadRTCP(); readErr != nil { + return + } + } + }() + } + return transport, nil +} + +func (transport *Transport) SetConnected() error { + transport.mu.Lock() + defer transport.mu.Unlock() + if transport.closed { + return io.ErrClosedPipe + } + if transport.attached { + return nil + } + receiver := transport.Output.Receiver().(*outputReceiver) + receiver.attached = true + if err := receiver.AddDownTrack(transport.Output); err != nil { + receiver.attached = false + return err + } + transport.attached = true + transport.Output.SetConnected() + transport.Output.Reconcile() + return nil +} + +func (transport *Transport) CurrentSource() *Source { + transport.mu.Lock() + defer transport.mu.Unlock() + return transport.source +} + +// ReplaceSource retains the connection and its congestion state. The caller +// prepares recovery in next and serializes group lifetime with this handoff. +func (transport *Transport) ReplaceSource(next *Source) error { + recovery := false + // Register first so feedback runs after every lock below has been released. + defer func() { + if recovery { + for layer := range next.TrackInfo().Layers { + next.SendPLI(int32(layer), true) + } + } + }() + transport.mu.Lock() + defer transport.mu.Unlock() + if transport.closed || transport.Output.IsClosed() { + return io.ErrClosedPipe + } + if next == nil { + return errors.New("replacement video source is absent") + } + next.mu.Lock() + defer next.mu.Unlock() + if err := validateReplacement(transport.source, next); err != nil { + return err + } + if next == transport.source { + return nil + } + receiver := &outputReceiver{Source: next, output: transport.Output, attached: transport.attached} + if err := transport.Output.replaceReceiver(receiver); err != nil { + return err + } + transport.source = next + recovery = transport.attached + return nil +} + +// next.mu prevents source closure between validation and receiver attachment. +func validateReplacement(current, next *Source) error { + if next.closed || next.IsClosed() { + return io.ErrClosedPipe + } + before, after := current.Codec(), next.Codec() + if before.PayloadType != after.PayloadType || before.MimeType != after.MimeType || + before.ClockRate != after.ClockRate || before.Channels != after.Channels || + before.SDPFmtpLine != after.SDPFmtpLine || !slices.Equal(before.RTCPFeedback, after.RTCPFeedback) || + current.TrackID() != next.TrackID() || current.StreamID() != next.StreamID() || + current.VideoLayerMode() != next.VideoLayerMode() || current.maxPackets != next.maxPackets || + len(current.TrackInfo().Layers) != len(next.TrackInfo().Layers) || + !slices.Equal(current.HeaderExtensions(), next.HeaderExtensions()) { + return errors.New("replacement video source changed its transport contract") + } + return nil +} + +func (transport *Transport) TargetBitrate() (int, bool) { + return int(transport.target.Load()), transport.observed.Load() +} + +func (transport *Transport) RequiredActiveCount() int { + state := transport.Output.State() + highest := state.Target + if !state.Paused { + highest = max(highest, state.Current) + } + if transport.observed.Load() { + highest = max(highest, state.Prepare) + } + if highest < 0 && !state.Paused { + highest = transport.Output.MaxLayer().Spatial + } + return int(highest + 1) +} + +// Egress counts successful RTP writes, including headers but excluding SRTP/UDP +// overhead. It is local delivery to Pion, not proof of remote receipt or decode. +func (transport *Transport) Egress() EgressStats { + return transport.pacer.stats() +} + +// SetAudioBitrate reserves the current audio owner's contribution to this PC's +// shared estimate. Zero means no audio demand; the transport invents no profile. +func (transport *Transport) SetAudioBitrate(bitrate uint32) { + transport.mu.Lock() + defer transport.mu.Unlock() + if !transport.closed { + transport.audioBitrate = bitrate + transport.refreshBudgetLocked() + } +} + +func (transport *Transport) refreshBudgetLocked() { + bitrate := int(transport.target.Load()) + remaining := int64(bitrate) - int64(transport.audioBitrate) + if transport.probe.Id == ccutils.ProbeClusterIdInvalid || transport.probe.Result.EndTime != 0 { + transport.pacer.SetBitrate(bitrate) + } + transport.Output.SetBudget(max(remaining, 0)) +} + +func (transport *Transport) Close() error { + transport.closeOnce.Do(func() { + transport.mu.Lock() + transport.closed = true + attached := transport.attached + transport.stopProbeLocked() + transport.mu.Unlock() + if transport.PC != nil { + transport.closeErr = transport.PC.Close() + } + if transport.Output != nil { + if attached { + transport.source.DeleteDownTrack(transport.Output.SubscriberID()) + } + transport.Output.Close() + } + if transport.pacer != nil { + transport.pacer.Stop() + } + transport.readers.Wait() + }) + return transport.closeErr +} + +// LiveKit delivers feedback without a ReadRTCP loop on its callback reader. +func (transport *Transport) OnTransportCCFeedback(_ *sfu.DownTrack, packet *rtcp.TransportLayerCC) { + transport.mu.Lock() + defer transport.mu.Unlock() + if !transport.closed { + transport.estimator.HandleTWCCFeedback(packet) + transport.observed.Store(true) + transport.refreshBudgetLocked() + transport.driveProbeLocked() + } +} + +// HandleTWCCFeedback invokes this synchronously under the transport's event +// lock. Store the framework result without reentering that lock. +func (transport *Transport) OnCongestionStateChange(_, _ bwe.CongestionState, capacity int64) { + if capacity > 0 { + transport.target.Store(capacity) + } +} + +func (*Transport) OnREMB(*sfu.DownTrack, *rtcp.ReceiverEstimatedMaximumBitrate) {} +func (*Transport) OnAvailableLayersChanged(*sfu.DownTrack) {} +func (*Transport) OnBitrateAvailabilityChanged(*sfu.DownTrack) {} +func (*Transport) OnMaxPublishedSpatialChanged(*sfu.DownTrack) {} +func (*Transport) OnMaxPublishedTemporalChanged(*sfu.DownTrack) {} +func (*Transport) OnSubscribedLayerChanged(*sfu.DownTrack, buffer.VideoLayer) {} +func (*Transport) OnResume(*sfu.DownTrack) {} +func (*Transport) IsBWEEnabled(*sfu.DownTrack) bool { return true } +func (*Transport) BWEType() bwe.BWEType { return bwe.BWETypeSendSide } +func (*Transport) IsSubscribeMutable(*sfu.DownTrack) bool { return true } +func (transport *Transport) OnSubscriptionChanged(*sfu.DownTrack) { transport.Output.Reconcile() } +func (*Transport) OnBindAndConnected() {} +func (*Transport) OnStatsUpdate(*livekit.AnalyticsStat) {} +func (*Transport) OnMaxSubscribedLayerChanged(int32) {} +func (transport *Transport) OnRttUpdate(rtt uint32) { + transport.mu.Lock() + defer transport.mu.Unlock() + if !transport.closed { + transport.estimator.UpdateRTT(float64(rtt) / 1000) + } +} +func (*Transport) OnCodecNegotiated(webrtc.RTPCodecCapability) {} +func (*Transport) OnDownTrackClose(bool) {} +func (*Transport) OnStreamStarted(time.Duration) {} + +// Probe orchestration below adapts the single-track flow from LiveKit v1.13.6: +// https://github.com/livekit/livekit/blob/v1.13.6/pkg/sfu/streamallocator/streamallocator.go +// Copyright 2023 LiveKit, Inc. +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 +// Unless required by applicable law or agreed to in writing, software distributed +// under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR +// CONDITIONS OF ANY KIND, either express or implied. See the License for the +// specific language governing permissions and limitations under the License. + +func (transport *Transport) mediaActivity() { + transport.mu.Lock() + defer transport.mu.Unlock() + transport.driveProbeLocked() +} + +func (transport *Transport) driveProbeLocked() { + if transport.closed || !transport.attached || transport.PC.ConnectionState() != webrtc.PeerConnectionStateConnected { + transport.paddingReady = false + transport.stopProbeLocked() + return + } + if transport.probe.Id != ccutils.ProbeClusterIdInvalid { + if transport.probe.Result.EndTime == 0 && (!transport.paddingReady || + transport.estimator.CongestionState() != bwe.CongestionStateNone || transport.estimator.ProbeClusterIsGoalReached() || + time.Since(transport.probe.CreatedAt) >= transport.probe.Goal.Duration) { + transport.stopProbeLocked() + } + if transport.probe.Result.EndTime != 0 { + if signal, capacity, finalized := transport.estimator.ProbeClusterFinalize(); finalized { + valid := sendsidebwe.DefaultSendSideBWEConfig.CongestionDetector.ProbeSignal.IsValid(transport.probe) + if valid && signal == ccutils.ProbeSignalNotCongesting && capacity > transport.target.Load() { + transport.target.Store(capacity) + } + transport.probe = ccutils.ProbeClusterInfoInvalid + transport.refreshBudgetLocked() + } + } + } + config := streamallocator.DefaultStreamAllocatorConfig + if transport.Output.State().Paused && transport.estimator.CongestionState() != bwe.CongestionStateNone && + transport.pacer.TimeSinceLastSentPacket() > config.PausedMinWait { + transport.stopProbeLocked() + transport.estimator.Reset() + transport.probe = ccutils.ProbeClusterInfoInvalid + } + if transport.probe.Id != ccutils.ProbeClusterIdInvalid || !transport.paddingReady || + !transport.observed.Load() || !transport.estimator.CanProbe() { + return + } + delta, usage := transport.Output.probeDemand() + if delta <= 0 { + return + } + usage += int64(transport.audioBitrate) + goal := ccutils.ProbeClusterGoal{AvailableBandwidthBps: int(transport.target.Load()), + ExpectedUsageBps: int(usage), DesiredBps: int(usage + max(delta*config.ProbeOveragePct/100, config.ProbeMinBps)), + Duration: transport.estimator.ProbeDuration()} + transport.probe = transport.prober.AddCluster(ccutils.ProbeClusterModeUniform, goal) + transport.estimator.ProbeClusterStarting(transport.probe) + transport.pacer.StartProbeCluster(transport.probe) + transport.pacer.SetBitrate(goal.DesiredBps) + transport.Output.SetProbeClusterId(transport.probe.Id) +} + +func (transport *Transport) stopProbeLocked() { + if transport.probe.Id == ccutils.ProbeClusterIdInvalid || transport.probe.Result.EndTime != 0 { + return + } + transport.probe = transport.pacer.EndProbeCluster(transport.probe.Id) + transport.Output.SwapProbeClusterId(transport.probe.Id, ccutils.ProbeClusterIdInvalid) + transport.estimator.ProbeClusterDone(transport.probe) + transport.prober.Reset(transport.probe) + transport.pacer.cancelProbes(transport.probe.Id) + transport.pacer.SetBitrate(int(transport.target.Load())) +} + +// AddCluster returns the exact cluster synchronously. Initializing it there +// avoids reentering the Prober while its own cluster-start mutex is held. +func (*Transport) OnProbeClusterSwitch(ccutils.ProbeClusterInfo) {} + +func (transport *Transport) OnSendProbe(bytesToSend int) { + transport.mu.Lock() + defer transport.mu.Unlock() + transport.driveProbeLocked() + if transport.probe.Id == ccutils.ProbeClusterIdInvalid || transport.probe.Result.EndTime != 0 { + return + } + written := transport.Output.WriteProbePackets(bytesToSend, true) + transport.prober.ProbesSent(written) + // Zero can be a temporary non-frame-boundary. The library-supplied cluster + // duration bounds retries; only connection/RR authority controls readiness. +} + +func (transport *Transport) OnPacerProbeObserverClusterComplete(id ccutils.ProbeClusterId) { + transport.mu.Lock() + defer transport.mu.Unlock() + if transport.probe.Id == id { + transport.stopProbeLocked() + } +} + +type EgressStats struct { + // Drops include canceled probe padding and queued packets retired at close. + Packets, RTPBytes, DroppedPackets uint64 +} + +// LiveKit's queue has no capacity limit. Only these bounded tickets enter it; +// the original packet remains ours until a ticket sends or Stop retires it. +type boundedPacer struct { + pacer.Pacer + sender *pacer.Base + recycle *pacer.Base + mu sync.Mutex + pending map[*pacedPacket]struct{} + limit int + closed bool + packets atomic.Uint64 + bytes atomic.Uint64 + dropped atomic.Uint64 + last atomic.Int64 +} + +func newBoundedPacer(packetPacer pacer.Pacer, log logger.Logger, limit int) *boundedPacer { + return &boundedPacer{Pacer: packetPacer, sender: pacer.NewBase(log, nil), recycle: pacer.NewBase(log, nil), + pending: make(map[*pacedPacket]struct{}), limit: limit} +} + +func (queue *boundedPacer) SetBitrate(bitrate int) { + // Pion interceptor v0.1.47 gcc.LeakyBucketPacer.SetTargetBitrate uses 1.5 + // pacing headroom. Codec/allocation budgets retain the unscaled BWE result. + queue.Pacer.SetBitrate(int(1.5 * float64(bitrate))) +} + +func (queue *boundedPacer) Enqueue(packet *pacer.Packet) { + queue.mu.Lock() + if queue.closed || len(queue.pending) >= queue.limit { + queue.mu.Unlock() + queue.dropped.Add(1) + queue.discard(packet) + return + } + ticket := &pacedPacket{owner: queue, packet: packet} + queue.pending[ticket] = struct{}{} + proxy := pacer.PacketFactory.Get().(*pacer.Packet) + // The library uses the byte count to pace; it never owns a reference to the + // pooled payload/header that Stop can release before this ticket is visited. + *proxy = pacer.Packet{HeaderSize: packet.HeaderSize + len(packet.Payload), WriteStream: ticket} + queue.mu.Unlock() + queue.Pacer.Enqueue(proxy) +} + +func (queue *boundedPacer) discard(packet *pacer.Packet) { + packet.WriteStream = closedPacketWriter{} + // Base owns all header/payload/packet pool returns, including failed writes. + _, _ = queue.recycle.SendPacket(packet) +} + +func (queue *boundedPacer) cancelProbes(id ccutils.ProbeClusterId) { + queue.mu.Lock() + var packets []*pacer.Packet + for ticket := range queue.pending { + if ticket.packet != nil && ticket.packet.IsProbe && ticket.packet.ProbeClusterId == id { + packets = append(packets, ticket.packet) + ticket.packet = nil + // The header-free proxy still occupies the library queue. Keep its + // admission slot charged until that proxy is actually dequeued. + } + } + queue.mu.Unlock() + for _, packet := range packets { + queue.dropped.Add(1) + queue.discard(packet) + } +} + +func (queue *boundedPacer) SetPacerProbeObserverListener(listener pacer.PacerProbeObserverListener) { + queue.sender.SetPacerProbeObserverListener(listener) +} + +func (queue *boundedPacer) StartProbeCluster(info ccutils.ProbeClusterInfo) { + queue.sender.StartProbeCluster(info) +} + +func (queue *boundedPacer) EndProbeCluster(id ccutils.ProbeClusterId) ccutils.ProbeClusterInfo { + return queue.sender.EndProbeCluster(id) +} + +func (queue *boundedPacer) Stop() { + queue.mu.Lock() + if queue.closed { + queue.mu.Unlock() + return + } + queue.closed = true + packets := make([]*pacer.Packet, 0, len(queue.pending)) + for ticket := range queue.pending { + if ticket.packet != nil { + packets = append(packets, ticket.packet) + } + ticket.packet = nil + } + clear(queue.pending) + queue.Pacer.Stop() + queue.mu.Unlock() + for _, packet := range packets { + queue.dropped.Add(1) + queue.discard(packet) + } +} + +func (queue *boundedPacer) TimeSinceLastSentPacket() time.Duration { + return time.Since(time.Unix(0, queue.last.Load())) +} + +func (queue *boundedPacer) stats() EgressStats { + return EgressStats{Packets: queue.packets.Load(), RTPBytes: queue.bytes.Load(), DroppedPackets: queue.dropped.Load()} +} + +type pacedPacket struct { + owner *boundedPacer + packet *pacer.Packet +} + +func (ticket *pacedPacket) WriteRTP(*rtp.Header, []byte) (int, error) { + queue := ticket.owner + queue.mu.Lock() + packet := ticket.packet + ticket.packet = nil + delete(queue.pending, ticket) + queue.mu.Unlock() + if packet == nil { + return 0, io.ErrClosedPipe + } + written, err := queue.sender.SendPacket(packet) + if err == nil && written > 0 { + queue.packets.Add(1) + queue.bytes.Add(uint64(written)) + queue.last.Store(time.Now().UnixNano()) + } + return written, err +} + +func (*pacedPacket) Write([]byte) (int, error) { return 0, io.ErrClosedPipe } + +type closedPacketWriter struct{} + +func (closedPacketWriter) WriteRTP(*rtp.Header, []byte) (int, error) { return 0, io.ErrClosedPipe } +func (closedPacketWriter) Write([]byte) (int, error) { return 0, io.ErrClosedPipe } diff --git a/internal/media/forwarding/transport_test.go b/internal/media/forwarding/transport_test.go new file mode 100644 index 0000000..2cb4eda --- /dev/null +++ b/internal/media/forwarding/transport_test.go @@ -0,0 +1,438 @@ +package forwarding + +import ( + "context" + "net" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/livekit/livekit-server/pkg/sfu" + "github.com/livekit/livekit-server/pkg/sfu/ccutils" + "github.com/livekit/livekit-server/pkg/sfu/pacer" + "github.com/livekit/protocol/logger" + "github.com/pion/interceptor" + "github.com/pion/rtcp" + "github.com/pion/rtp" + "github.com/pion/rtp/codecs" + "github.com/pion/webrtc/v4" + "github.com/pion/webrtc/v4/pkg/media" +) + +func TestTransportVideoFeedbackAudioAndClose(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 5*time.Second) + defer cancel() + check := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + codec := webrtc.RTPCodecParameters{RTPCodecCapability: webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeVP8, ClockRate: 90_000, + RTCPFeedback: []webrtc.RTCPFeedback{{Type: "nack"}, {Type: "nack", Parameter: "pli"}, {Type: webrtc.TypeRTCPFBTransportCC}}, + }, PayloadType: 96} + keyFrames := make(chan struct{}, 8) + source, err := NewSource(SourceOptions{ID: "screen", StreamID: "fixture", Codec: codec, + Formats: []LayerFormat{{Width: 8, Height: 8, Bitrate: 300_000}, {Width: 16, Height: 16, Bitrate: 3_000_000}}, OnRTCP: func(_ int, packets []rtcp.Packet) { + for _, packet := range packets { + if _, ok := packet.(*rtcp.PictureLossIndication); ok { + select { + case keyFrames <- struct{}{}: + default: + } + } + } + }}) + check(err) + defer source.Close() + check(source.BindLayer(0, 1000, webrtc.RTPParameters{Codecs: []webrtc.RTPCodecParameters{codec}})) + // This checks feedback routing independently of the buffer's bootstrap-PLI throttle. + source.buffers[0].SetPLIThrottle(0) + settings := webrtc.SettingEngine{} + settings.SetNetworkTypes([]webrtc.NetworkType{webrtc.NetworkTypeUDP4}) + settings.SetIncludeLoopbackCandidate(true) + settings.SetIPFilter(func(ip net.IP) bool { return ip.IsLoopback() }) + audio, err := webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeOpus, ClockRate: 48_000, Channels: 2, + }, "audio", "fixture") + check(err) + demandWake := make(chan struct{}, 1) + transport, err := NewTransport(TransportOptions{Source: source, Settings: settings, + ConnectionID: "child", InitialBitrate: 1_000_000, Audio: audio, OnDemandChanged: func() { + select { + case demandWake <- struct{}{}: + default: + } + }}) + check(err) + defer transport.Close() + transport.Output.SetMaxSpatialLayer(0) + transport.SetAudioBitrate(64_000) + if state := transport.Output.State(); state.VideoBudget != 936_000 { + t.Fatalf("audio reservation did not reach the video codec budget: %+v", state) + } + if len(source.GetDownTracks()) != 0 { + t.Fatal("unconnected transport claimed source output") + } + if transport.RequiredActiveCount() != 1 { + t.Fatal("initial configured acquisition demand was lost") + } + mediaEngine := &webrtc.MediaEngine{} + check(mediaEngine.RegisterDefaultCodecs()) + registry := &interceptor.Registry{} + check(webrtc.RegisterDefaultInterceptors(mediaEngine, registry)) + check(webrtc.ConfigureTWCCSender(mediaEngine, registry)) + reports := &receiverReportGate{} + registry.Add(reports) + receiver, err := webrtc.NewAPI(webrtc.WithMediaEngine(mediaEngine), webrtc.WithInterceptorRegistry(registry), + webrtc.WithSettingEngine(settings)).NewPeerConnection(webrtc.Configuration{}) + check(err) + defer receiver.Close() + var videoPackets, audioPackets atomic.Uint32 + firstVideo := make(chan *rtp.Packet, 1) + var recoveryTimestamp atomic.Uint32 + recovered := make(chan struct{}, 1) + var readers sync.WaitGroup + receiver.OnTrack(func(track *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { + readers.Add(1) + defer readers.Done() + for { + packet, _, readErr := track.ReadRTP() + if readErr != nil { + return + } + if len(packet.Payload) == 0 { + continue + } + if track.Kind() == webrtc.RTPCodecTypeVideo { + videoPackets.Add(1) + select { + case firstVideo <- packet: + default: + } + if packet.Timestamp == recoveryTimestamp.Load() { + select { + case recovered <- struct{}{}: + default: + } + } + } else { + audioPackets.Add(1) + } + } + }) + offer, err := transport.PC.CreateOffer(nil) + check(err) + gather := webrtc.GatheringCompletePromise(transport.PC) + check(transport.PC.SetLocalDescription(offer)) + select { + case <-gather: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + check(receiver.SetRemoteDescription(*transport.PC.LocalDescription())) + answer, err := receiver.CreateAnswer(nil) + check(err) + gather = webrtc.GatheringCompletePromise(receiver) + check(receiver.SetLocalDescription(answer)) + select { + case <-gather: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + check(transport.PC.SetRemoteDescription(*receiver.LocalDescription())) + for transport.PC.ConnectionState() != webrtc.PeerConnectionStateConnected { + select { + case <-time.After(5 * time.Millisecond): + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + } + check(transport.SetConnected()) + check(transport.SetConnected()) + if len(source.GetDownTracks()) != 1 { + t.Fatal("connected transport attached twice") + } + packetizer := rtp.NewPacketizer(1200, 96, 1000, &codecs.VP8Payloader{EnablePictureID: true}, rtp.NewRandomSequencer(), 90_000) + for _, packet := range packetizer.Packetize(sfu.VP8KeyFrame8x8, 3000) { + check(source.WriteRTP(0, packet)) + } + firstFrame := false + for !firstFrame { + select { + case packet := <-firstVideo: + firstFrame = true + recoveryTimestamp.Store(packet.Timestamp + 3000) + for len(keyFrames) > 0 { + <-keyFrames + } + check(receiver.WriteRTCP([]rtcp.Packet{&rtcp.PictureLossIndication{MediaSSRC: packet.SSRC}})) + case <-time.After(time.Second / 30): + for _, packet := range packetizer.Packetize(sfu.VP8KeyFrame8x8, 3000) { + check(source.WriteRTP(0, packet)) + } + case <-ctx.Done(): + t.Fatalf("missing first frame: %+v", transport.Output.State()) + } + } + select { + case <-keyFrames: + case <-time.After(time.Second): + t.Fatalf("first-frame PLI never reached source; output=%+v statsPLI=%d", transport.Output.State(), transport.Output.GetTrackStats().Plis) + } + // Force exactly one local queue-admission loss. The following RTP packet + // exposes its sequence gap to the receiver's normal NACK generator. + transport.pacer.mu.Lock() + transport.pacer.limit = 0 + transport.pacer.mu.Unlock() + for _, packet := range packetizer.Packetize(sfu.VP8KeyFrame8x8, 3000) { + check(source.WriteRTP(0, packet)) + } + for transport.Egress().DroppedPackets == 0 { + select { + case <-time.After(5 * time.Millisecond): + case <-ctx.Done(): + t.Fatal("local admission loss was not exercised") + } + } + transport.pacer.mu.Lock() + transport.pacer.limit = source.maxPackets + transport.pacer.mu.Unlock() + for _, packet := range packetizer.Packetize(sfu.VP8KeyFrame8x8, 3000) { + check(source.WriteRTP(0, packet)) + } + select { + case <-recovered: + case <-ctx.Done(): + t.Fatal("local admission loss did not recover through library NACK") + } + ticker := time.NewTicker(time.Second / 30) + defer ticker.Stop() + for { + _, observed := transport.TargetBitrate() + if videoPackets.Load() >= 15 && audioPackets.Load() >= 15 && observed { + break + } + select { + case <-ctx.Done(): + t.Fatalf("media or feedback missing: video=%d audio=%d observed=%v", videoPackets.Load(), audioPackets.Load(), observed) + case <-ticker.C: + } + for _, packet := range packetizer.Packetize(sfu.VP8KeyFrame8x8, 3000) { + check(source.WriteRTP(0, packet)) + } + check(audio.WriteSample(media.Sample{Data: []byte{0xf8, 0xff, 0xfe}, Duration: 20 * time.Millisecond})) + } + if egress := transport.Egress(); egress.Packets < 15 || egress.RTPBytes == 0 || egress.DroppedPackets != 1 { + t.Fatalf("paced local egress disagrees with receiving media: %+v", egress) + } + if len(demandWake) == 0 || transport.RequiredActiveCount() != 1 { + t.Fatal("typed direct-child demand was not signaled") + } + // Raise subscribed demand without inventing an available layer. TWCC alone + // cannot establish the RR readiness required by the library padding sender. + transport.Output.DownTrack.SetMaxSpatialLayer(1) + if delta, _ := transport.Output.probeDemand(); delta <= 0 { + t.Fatal("no-RR fixture did not establish dormant upper demand") + } + transport.mediaActivity() + transport.mu.Lock() + ready := transport.paddingReady + transport.mu.Unlock() + if ready || transport.prober.IsRunning() { + t.Fatal("transport probed before a receiver report") + } + // A muted library sender returns zero padding bytes despite a connected PC + // and valid RR. Its probe must finish, and inconclusive feedback must not + // promote the BWE's internal initial-capacity sentinel into a codec budget. + transport.Output.Mute(true) + before, _ := transport.TargetBitrate() + reports.allow.Store(true) + seenProbe := false + for { + transport.mediaActivity() + transport.mu.Lock() + active := transport.probe.Id != ccutils.ProbeClusterIdInvalid + transport.mu.Unlock() + seenProbe = seenProbe || active + if seenProbe && !active && !transport.prober.IsRunning() { + break + } + select { + case <-time.After(5 * time.Millisecond): + case <-ctx.Done(): + t.Fatal("zero-write probe did not retire within its library duration/settle bound") + } + } + if after, _ := transport.TargetBitrate(); after != before { + t.Fatalf("zero-write probe manufactured a budget: before=%d after=%d", before, after) + } + check(transport.Close()) + check(transport.Close()) + check(receiver.Close()) + readers.Wait() + if len(source.GetDownTracks()) != 0 || transport.SetConnected() == nil || transport.RequiredActiveCount() != 0 { + t.Fatal("closed transport retained a source subscription") + } +} + +type receiverReportGate struct { + interceptor.NoOp + allow atomic.Bool +} + +func (gate *receiverReportGate) NewInterceptor(string) (interceptor.Interceptor, error) { + return gate, nil +} + +func (gate *receiverReportGate) BindRTCPWriter(writer interceptor.RTCPWriter) interceptor.RTCPWriter { + return interceptor.RTCPWriterFunc(func(packets []rtcp.Packet, attributes interceptor.Attributes) (int, error) { + if gate.allow.Load() { + return writer.Write(packets, attributes) + } + var filtered []rtcp.Packet + for _, packet := range packets { + if _, report := packet.(*rtcp.ReceiverReport); !report { + filtered = append(filtered, packet) + } + } + if len(filtered) == 0 { + return 0, nil + } + return writer.Write(filtered, attributes) + }) +} + +type heldPacketPacer struct { + pacer.Pacer + packets []*pacer.Packet +} + +func (held *heldPacketPacer) Enqueue(packet *pacer.Packet) { + held.packets = append(held.packets, packet) +} + +type countingPacketWriter struct{ bytes int } + +func (writer *countingPacketWriter) WriteRTP(header *rtp.Header, payload []byte) (int, error) { + written := header.MarshalSize() + len(payload) + writer.bytes += written + return written, nil +} + +func (*countingPacketWriter) Write([]byte) (int, error) { panic("unexpected byte write") } + +func TestBoundedPacerReleasesPacketsAndIsolatesBlockedQueue(t *testing.T) { + log := logger.GetDiscardLogger() + sender := pacer.NewPassThrough(log, nil) + held := &heldPacketPacer{Pacer: sender} + queue := newBoundedPacer(held, log, 2) + defer queue.Stop() + healthy := newBoundedPacer(sender, log, 2) + defer healthy.Stop() + writer := &countingPacketWriter{} + var originals []*pacer.Packet + packet := func() *pacer.Packet { + header := &rtp.Header{Version: 2, SSRC: 1} + payload := make([]byte, 100) + original := &pacer.Packet{Header: header, HeaderSize: header.MarshalSize(), + HeaderPool: &sync.Pool{}, Pool: &sync.Pool{}, PoolEntity: &payload, + Payload: payload, WriteStream: writer} + originals = append(originals, original) + return original + } + queue.Enqueue(packet()) + queue.Enqueue(packet()) + queue.Enqueue(packet()) + if len(queue.pending) != 2 || queue.stats().DroppedPackets != 1 || len(held.packets) != 2 { + t.Fatalf("queue exceeded its bound: pending=%d stats=%+v", len(queue.pending), queue.stats()) + } + healthy.Enqueue(packet()) + if healthy.stats().Packets != 1 || queue.stats().Packets != 0 { + t.Fatal("held sibling blocked a ready packet") + } + sender.Enqueue(held.packets[0]) + queue.Enqueue(packet()) + queue.Stop() + queue.Stop() + queue.Enqueue(packet()) + // Upstream may still hold header-free tickets when it stops. Visiting one + // after cancellation must neither send nor return the original a second time. + for _, proxy := range held.packets[1:] { + sender.Enqueue(proxy) + } + if stats := queue.stats(); stats != (EgressStats{Packets: 1, RTPBytes: 112, DroppedPackets: 4}) { + t.Fatalf("incorrect sent/drop accounting: %+v", stats) + } + for _, original := range originals { + if original.Header != nil || original.Payload != nil || original.WriteStream != nil || original.PoolEntity != nil { + t.Fatalf("send/Stop retained pooled packet ownership: %+v", original) + } + } + if writer.bytes != 224 { + t.Fatalf("canceled packets reached the writer: %d bytes", writer.bytes) + } +} + +func TestBoundedPacerCanceledProbesRetainAdmission(t *testing.T) { + log := logger.GetDiscardLogger() + sender := pacer.NewPassThrough(log, nil) + held := &heldPacketPacer{Pacer: sender} + queue := newBoundedPacer(held, log, 2) + defer queue.Stop() + packet := func() *pacer.Packet { + return &pacer.Packet{Header: &rtp.Header{Version: 2}, HeaderSize: 12, + Payload: make([]byte, 100), WriteStream: &countingPacketWriter{}, IsProbe: true, ProbeClusterId: 1} + } + queue.Enqueue(packet()) + queue.Enqueue(packet()) + for range 10 { + queue.cancelProbes(1) + queue.Enqueue(packet()) + } + if len(queue.pending) != 2 || len(held.packets) != 2 || queue.stats().DroppedPackets != 12 { + t.Fatalf("canceled proxies freed admission before dequeue: pending=%d queued=%d stats=%+v", + len(queue.pending), len(held.packets), queue.stats()) + } + sender.Enqueue(held.packets[0]) + queue.Enqueue(packet()) + if len(queue.pending) != 2 || len(held.packets) != 3 { + t.Fatal("actual dequeue did not return its admission slot") + } + queue.Stop() + for _, proxy := range held.packets[1:] { + sender.Enqueue(proxy) + } + if stats := queue.stats(); stats.Packets != 0 || stats.DroppedPackets != 13 { + t.Fatalf("canceled ownership was sent or retired twice: %+v", stats) + } +} + +func BenchmarkBoundedPacerAdmission(b *testing.B) { + log := logger.GetDiscardLogger() + for _, bounded := range []bool{false, true} { + name := "pass-through" + if bounded { + name = "bounded-admission" + } + b.Run(name, func(b *testing.B) { + var queue pacer.Pacer = pacer.NewPassThrough(log, nil) + if bounded { + queue = newBoundedPacer(queue, log, 500) + } + defer queue.Stop() + writer := &countingPacketWriter{} + header := &rtp.Header{Version: 2, SSRC: 1} + payload := make([]byte, 1200) + b.ReportAllocs() + b.ResetTimer() + for b.Loop() { + packet := pacer.PacketFactory.Get().(*pacer.Packet) + *packet = pacer.Packet{Header: header, HeaderSize: header.MarshalSize(), Payload: payload, WriteStream: writer} + queue.Enqueue(packet) + } + }) + } +} diff --git a/internal/server/app/access.go b/internal/server/app/access.go index 2a4b30a..7eb9443 100644 --- a/internal/server/app/access.go +++ b/internal/server/app/access.go @@ -6,11 +6,14 @@ import ( "crypto/subtle" "encoding/base64" "errors" + "net/http" + "net/url" "regexp" "strconv" "strings" "time" + "github.com/TNTcraftHIM/Screener/internal/server/config" "github.com/TNTcraftHIM/Screener/internal/server/protocol" ) @@ -47,6 +50,25 @@ type siteAccessGate struct { ttlSeconds int } +// One Local authority can be served through HTTPS public links and HTTP LAN +// origins. Select the cookie contract by its configured destination, never by +// the initiating Origin or an untrusted forwarded-protocol header. +func (s *Server) siteAccessForRequest(request *http.Request) *siteAccessGate { + if !s.siteAccess.secure || request.TLS != nil { + return s.siteAccess + } + plain := config.Origin(&url.URL{Scheme: "http", Host: request.Host}) + secure := config.Origin(&url.URL{Scheme: "https", Host: request.Host}) + _, allowsPlain := s.config.AllowedOrigins[plain] + _, allowsSecure := s.config.AllowedOrigins[secure] + if !allowsPlain || allowsSecure || strings.EqualFold(request.Host, s.config.PublicBaseURL.Host) { + return s.siteAccess + } + access := *s.siteAccess + access.secure = false + return &access +} + // newSiteAccess mirrors the SiteAccess constructor, including its TTL check. func newSiteAccess(options siteAccessOptions) (*siteAccessGate, error) { ttlSeconds := options.TTLSeconds diff --git a/internal/server/app/app.go b/internal/server/app/app.go index 88b639b..0b406d7 100644 --- a/internal/server/app/app.go +++ b/internal/server/app/app.go @@ -34,6 +34,9 @@ import ( "github.com/TNTcraftHIM/Screener/internal/server/room" "github.com/TNTcraftHIM/Screener/internal/server/sfu" "github.com/TNTcraftHIM/Screener/internal/server/signal" + "github.com/TNTcraftHIM/Screener/internal/server/stun" + "github.com/pion/ice/v4" + "github.com/pion/webrtc/v4" ) // Node HTTP server settings of app.ts, mapped in map section 3.3 / D11. @@ -47,6 +50,9 @@ const ( type Options struct { // Config is the loaded server configuration; cmd owns config.Load / Local. Config config.Config + // Listener is optional prebound TCP ownership transferred on successful New. + // It lets the Local Client reserve its HTTP port before opening a tunnel. + Listener *net.TCPListener // Assets is the built Browser UI. A nil FS serves the API only, which is // the TS frontend mode "none"; webassets.FS() returns nil the same way // when no build was embedded (D3). @@ -64,10 +70,6 @@ type Options struct { MaxUnauthenticatedSignalConnections int SiteAccessTTLSeconds int - // SFUTokenIssuer and SFURoomControl replace the LiveKit implementations - // this package would otherwise construct from Config.LiveKit. - SFUTokenIssuer sfu.TokenIssuer - SFURoomControl sfu.RoomControl // AfterFunc is the signaling timer factory (D5); nil uses time.AfterFunc. AfterFunc func(time.Duration, func()) func() bool // RoomStore replaces the store this package would build from Config. @@ -76,19 +78,20 @@ type Options struct { // Server is ScreenerServer. type Server struct { - config config.Config - store *room.Store - siteAccess *siteAccessGate - frontend http.Handler - logger *slog.Logger - // roomControl is nil without a LiveKit fallback; it is the only startup - // reconciliation step (A12). - roomControl sfu.RoomControl + config config.Config + store *room.Store + siteAccess *siteAccessGate + frontend http.Handler + logger *slog.Logger + media *sfu.Media + mediaMux ice.UDPMux signalOptions signal.Options httpServer *http.Server - // listener is nil until Listen binds. It is written before startupDone is - // closed and read only after that channel settles, so it needs no lock. + // Listen binds unless New received an already owned listener. Shutdown waits + // for startupDone before reading it; closing before Listen also releases it. listener net.Listener + // STUN shares the same startup/shutdown owner, never the advertised ICE URLs. + stunServer *stun.Server // acceptingTraffic and signaling are the two closure variables app.ts // flipped together; both are read by request goroutines without a lock and @@ -121,10 +124,6 @@ func New(options Options) (*Server, error) { logger = slog.Default() } - roomControl, fallback, err := newSfuFallback(options) - if err != nil { - return nil, err - } store, err := newRoomStore(options, now) if err != nil { return nil, err @@ -146,16 +145,17 @@ func New(options Options) (*Server, error) { store: store, siteAccess: siteAccess, logger: logger, - roomControl: roomControl, startupDone: make(chan struct{}), } + if options.Listener != nil { + server.listener = options.Listener + } if options.Assets != nil { server.frontend = staticHandler(options.Assets, http.HandlerFunc(notFoundJSON)) } server.signalOptions = signal.Options{ Store: store, EndpointMediaCopyCapacity: configuration.EndpointMediaCopyCapacity, - SfuFallback: fallback, Ice: config.IceConfig(configuration), // The TS signaling server read natPredictionEnabled off the same // `ice` options object it forwarded; the Go IceConfig is already the @@ -163,7 +163,7 @@ func New(options Options) (*Server, error) { NATPredictionEnabled: configuration.NATPredictionEnabled, AllowedOrigins: configuration.AllowedOrigins, SiteAccessAtUpgrade: func(request *http.Request) bool { - return siteAccess.isAuthenticated(cookieHeader(request)) + return server.siteAccessForRequest(request).isAuthenticated(cookieHeader(request)) }, PublicBaseURL: configuration.PublicBaseURL, Now: now, @@ -186,51 +186,6 @@ func New(options Options) (*Server, error) { return server, nil } -// newSfuFallback builds the LiveKit half of createScreenerServer. Without a -// LiveKit fallback in the configuration both results are nil and the signaling -// server runs peer-only. The returned RoomControl is also kept by the caller, -// because it is the one startup reconciliation step (A12). -func newSfuFallback(options Options) (sfu.RoomControl, *signal.SfuFallback, error) { - livekit := options.Config.LiveKit - if livekit == nil { - return nil, nil, nil - } - roomControl := options.SFURoomControl - if roomControl == nil { - control, err := sfu.NewLiveKitRoomControl(sfu.LiveKitRoomControlOptions{ - APIURL: livekit.APIURL, - APIKey: livekit.APIKey, - APISecret: livekit.APISecret, - MaxViewersPerRoom: options.Config.MaxViewersPerRoom, - }) - if err != nil { - return nil, nil, err - } - roomControl = control - } - tokenIssuer := options.SFUTokenIssuer - if tokenIssuer == nil { - issuer, err := sfu.NewLiveKitTokenIssuer(sfu.LiveKitTokenIssuerOptions{ - APIKey: livekit.APIKey, - APISecret: livekit.APISecret, - MaxViewersPerRoom: options.Config.MaxViewersPerRoom, - }) - if err != nil { - return nil, nil, err - } - tokenIssuer = issuer - } - return roomControl, &signal.SfuFallback{ - URL: livekit.URL, - TokenIssuer: tokenIssuer, - Admission: sfu.NewAdmission(sfu.AdmissionOptions{ - IngressCapacity: room.Capacity, - EgressCapacity: room.Capacity * options.Config.MaxViewersPerRoom, - }), - RoomControl: roomControl, - }, nil -} - // newRoomStore builds the room store createScreenerServer would otherwise take // from Options.RoomStore. An empty RoomDatabasePath is the memory-only mode. func newRoomStore(options Options, now func() int64) (*room.Store, error) { @@ -263,9 +218,7 @@ func (s *Server) Handler() http.Handler { return s } // callers outside that lock may only read it while no request is in flight. func (s *Server) Store() *room.Store { return s.store } -// Listen binds the configured address, then reconciles the stable authority and -// LiveKit, and only then starts accepting traffic. It returns the bound port, -// which is the actual one when Config.Port is 0. +// Listen binds application sockets before recovering room authority and accepting traffic. func (s *Server) Listen(ctx context.Context) (int, error) { s.mu.Lock() if s.startupRequested { @@ -287,14 +240,48 @@ func (s *Server) Listen(ctx context.Context) (int, error) { } func (s *Server) start(ctx context.Context) (int, error) { - // A11: bind first. A listener owned by another process must fail before - // the room database is opened and before LiveKit is called at all. - address := net.JoinHostPort(s.config.ListenHost, strconv.Itoa(s.config.Port)) - listener, err := net.Listen("tcp", address) - if err != nil { - return 0, err + // Bind first. A listener owned by another process must fail before + // the room database is opened. + listener := s.listener + var err error + if listener == nil { + address := net.JoinHostPort(s.config.ListenHost, strconv.Itoa(s.config.Port)) + listener, err = net.Listen("tcp", address) + if err != nil { + return 0, err + } } s.listener = listener + if len(s.config.STUNListenAddresses) > 0 { + s.stunServer, err = stun.Listen(ctx, s.config.STUNListenAddresses) + if err != nil { + _ = listener.Close() + return 0, fmt.Errorf("Screener STUN listener failed: %w", err) + } + } + if configuration := s.config.SFU; configuration != nil { + connection, listenErr := net.ListenPacket("udp4", net.JoinHostPort(configuration.ListenHost, strconv.Itoa(configuration.Port))) + if listenErr != nil { + _ = s.stopServing(ctx) + return 0, fmt.Errorf("Screener SFU listener failed: %w", listenErr) + } + s.mediaMux = ice.NewUDPMuxDefault(ice.UDPMuxParams{UDPConn: connection}) + settings := webrtc.SettingEngine{} + settings.SetNetworkTypes([]webrtc.NetworkType{webrtc.NetworkTypeUDP4}) + settings.SetICEUDPMux(s.mediaMux) + settings.SetIncludeLoopbackCandidate(net.ParseIP(configuration.ListenHost).IsLoopback()) + if configuration.PublicIP != "" { + settings.SetNAT1To1IPs([]string{configuration.PublicIP}, webrtc.ICECandidateTypeHost) + } + s.media = sfu.NewMedia(sfu.MediaOptions{Settings: settings, Events: func(event sfu.MediaEvent) { + if signaling := s.signaling.Load(); signaling != nil { + signaling.HandleSfuMediaEvent(event) + } + }}) + s.signalOptions.SfuFallback = &signal.SfuFallback{Media: s.media, Admission: sfu.NewAdmission(sfu.AdmissionOptions{ + IngressCapacity: room.Capacity, EgressCapacity: room.Capacity * s.config.MaxViewersPerRoom, + })} + } go func() { // bindHttpServer removed its "error" listener once the server was // listening (app.ts:315-333), so a later listener failure ended the @@ -302,7 +289,7 @@ func (s *Server) start(ctx context.Context) (int, error) { // ErrServerClosed on a permanent accept failure, which nothing here // causes; report it rather than serving nothing in silence. if err := s.httpServer.Serve(listener); !errors.Is(err, http.ErrServerClosed) { - s.logger.Error("Screener HTTP server stopped unexpectedly", "error", err) + s.logger.Error("Screener HTTP server stopped unexpectedly", "errorType", fmt.Sprintf("%T", err)) } }() port := listener.Addr().(*net.TCPAddr).Port @@ -325,32 +312,37 @@ func (s *Server) start(ctx context.Context) (int, error) { return port, nil } -// stopServing is closeHttpServer. Shutdown stops accepting and drains, but it -// can lose the race against the Serve goroutine registering the listener, in +// stopServing releases the application-owned HTTP and STUN sockets. Shutdown +// stops accepting HTTP and drains, but it can lose the race against the Serve +// goroutine registering the listener, in // which case it returns without having closed the bound socket; closing the // listener afterwards makes the port release synchronous. A second close of an -// already closed listener is a no-op error nobody reported in TS either, so -// only Shutdown's error is returned. +// already closed HTTP listener is harmless. func (s *Server) stopServing(ctx context.Context) error { err := s.httpServer.Shutdown(ctx) if s.listener != nil { _ = s.listener.Close() } + if s.stunServer != nil { + err = errors.Join(err, s.stunServer.Close()) + s.stunServer = nil + } + if s.media != nil { + err = errors.Join(err, s.media.Close()) + s.media = nil + } + if s.mediaMux != nil { + err = errors.Join(err, s.mediaMux.Close()) + s.mediaMux = nil + } return err } -// reconcile is the body of the TS startup try block: the store recovers its -// stable authority synchronously, LiveKit reconciliation is the one await -// (A12), and traffic starts only if no shutdown began while it ran. +// reconcile restores room authority and enables traffic unless shutdown has begun. func (s *Server) reconcile(ctx context.Context) error { if err := s.store.Initialize(); err != nil { return err } - if s.roomControl != nil { - if err := s.roomControl.Initialize(ctx); err != nil { - return err - } - } // Holding mu across the check and the two stores is what made the TS // single thread safe here: a shutdown that already set closing must never // see traffic re-enabled behind it. @@ -368,7 +360,7 @@ func (s *Server) reconcile(ctx context.Context) error { return nil } -// Close stops accepting traffic, closes the signaling server, the listener and +// Close stops accepting traffic, closes signaling, the HTTP/STUN listeners and // the room store, and leaves the rooms in place. Hosted restarts use it. func (s *Server) Close(ctx context.Context) error { return s.shutdown(ctx, false) diff --git a/internal/server/app/app_test.go b/internal/server/app/app_test.go index df95d0d..d39a9eb 100644 --- a/internal/server/app/app_test.go +++ b/internal/server/app/app_test.go @@ -11,6 +11,7 @@ import ( "log/slog" "net" "net/http" + "net/http/cookiejar" "net/http/httptest" "net/url" "os" @@ -30,8 +31,6 @@ import ( "github.com/TNTcraftHIM/Screener/internal/server/config" "github.com/TNTcraftHIM/Screener/internal/server/protocol" "github.com/TNTcraftHIM/Screener/internal/server/room" - "github.com/TNTcraftHIM/Screener/internal/server/sfu" - "github.com/TNTcraftHIM/Screener/internal/server/sfu/sfutest" ) // The scenarios below are ported from tests/server-http.test.ts. @@ -46,12 +45,6 @@ var setCookiePattern = regexp.MustCompile( var secureSetCookiePattern = regexp.MustCompile( `^__Host-screener-site-access=v1\.[0-9]+\.[A-Za-z0-9_-]{43}; Path=/; Max-Age=86400; HttpOnly; SameSite=Strict; Secure$`) -type fakeTokenIssuer struct{} - -func (fakeTokenIssuer) IssueToken(sfu.TokenRequest) (string, error) { - return "unused-test-token", nil -} - func testConfig(t *testing.T) config.Config { t.Helper() publicBaseURL, err := url.Parse("https://share.example.test") @@ -72,15 +65,6 @@ func testConfig(t *testing.T) config.Config { } } -func livekitFallback() *config.LiveKitFallback { - return &config.LiveKitFallback{ - URL: "ws://livekit.test:7880", - APIURL: "http://127.0.0.1:7880", - APIKey: "test-key", - APISecret: strings.Repeat("s", 32), - } -} - // newServer builds a server without listening. It mirrors the TS harness: the // heartbeat and cleanup intervals are pushed out of the way and a fake room // control is injected whenever a LiveKit fallback is configured. @@ -95,14 +79,6 @@ func newServer(t *testing.T, options Options) *Server { if options.CleanupIntervalMs == 0 { options.CleanupIntervalMs = 60_000 } - if options.Config.LiveKit != nil { - if options.SFURoomControl == nil { - options.SFURoomControl = sfutest.New() - } - if options.SFUTokenIssuer == nil { - options.SFUTokenIssuer = fakeTokenIssuer{} - } - } server, err := New(options) if err != nil { t.Fatalf("New: %v", err) @@ -387,6 +363,7 @@ func TestCapabilitiesReportOptionalNATPredictionWithoutExposingConfiguration(t * configuration := testConfig(t) configuration.STUNURLs = []string{"stun:share.example.test:3478"} configuration.NATPredictionEnabled = true + configuration.STUNListenAddresses = []string{"127.0.0.1:0", "127.0.0.1:0", "127.0.0.1:0"} server := start(t, Options{Config: configuration}) server.do(http.MethodGet, "/api/capabilities"). @@ -399,6 +376,123 @@ func TestCapabilitiesReportOptionalNATPredictionWithoutExposingConfiguration(t * expectHeader("Allow", "GET") } +func TestSTUNListenersFollowApplicationCloseAndEnd(t *testing.T) { + for _, ending := range []bool{false, true} { + t.Run(fmt.Sprintf("end=%t", ending), func(t *testing.T) { + configuration := testConfig(t) + configuration.STUNURLs = []string{"stun:share.example.test:3478"} + configuration.NATPredictionEnabled = true + configuration.STUNListenAddresses = []string{"127.0.0.1:0", "127.0.0.1:0", "127.0.0.1:0"} + server := start(t, Options{Config: configuration}) + addresses := server.stunServer.Addresses + var err error + if ending { + err = server.End(context.Background()) + } else { + err = server.Close(context.Background()) + } + if err != nil { + t.Fatal(err) + } + assertUDPPortsFree(t, addresses) + }) + } +} + +func TestSFUListenerFollowsApplicationCloseAndEnd(t *testing.T) { + for _, ending := range []bool{false, true} { + t.Run(fmt.Sprintf("end=%t", ending), func(t *testing.T) { + configuration := testConfig(t) + configuration.SFU = &config.SFUConfig{ListenHost: "127.0.0.1", Port: 0} + server := start(t, Options{Config: configuration}) + addresses := server.mediaMux.GetListenAddresses() + if server.signaling.Load() == nil || server.media == nil { + t.Fatal("missing runtime owner") + } + var err error + if ending { + err = server.End(context.Background()) + } else { + err = server.Close(context.Background()) + } + if err != nil { + t.Fatal(err) + } + assertUDPPortsFree(t, addresses) + }) + } +} + +func TestOccupiedSFUPortRollsBackBeforeOpeningTheDatabase(t *testing.T) { + occupied, err := net.ListenPacket("udp4", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer occupied.Close() + configuration := testConfig(t) + configuration.Port = freePort(t) + configuration.RoomDatabasePath = filepath.Join(t.TempDir(), "rooms.sqlite") + configuration.STUNListenAddresses = []string{"127.0.0.1:0"} + configuration.SFU = &config.SFUConfig{ListenHost: "127.0.0.1", Port: occupied.LocalAddr().(*net.UDPAddr).Port} + server := newServer(t, Options{Config: configuration}) + if _, err = server.Listen(context.Background()); err == nil { + t.Fatal("occupied media listener was accepted") + } + if _, err = os.Stat(configuration.RoomDatabasePath); !os.IsNotExist(err) { + t.Fatalf("SFU bind failure touched room persistence: %v", err) + } + assertPortFree(t, configuration.Port) + if server.stunServer != nil || server.mediaMux != nil { + t.Fatal("failed startup retained a UDP listener") + } + if err = occupied.SetDeadline(time.Now().Add(time.Second)); err != nil { + t.Fatalf("startup rollback closed another listener: %v", err) + } +} + +func TestOccupiedSTUNAuxiliaryRollsBackBeforeOpeningTheDatabase(t *testing.T) { + first, err := net.ListenPacket("udp4", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + firstAddress := first.LocalAddr() + _ = first.Close() + occupied, err := net.ListenPacket("udp4", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer occupied.Close() + configuration := testConfig(t) + configuration.Port = freePort(t) + configuration.RoomDatabasePath = filepath.Join(t.TempDir(), "rooms.sqlite") + configuration.STUNListenAddresses = []string{ + firstAddress.String(), occupied.LocalAddr().String(), "127.0.0.1:0", + } + server := newServer(t, Options{Config: configuration}) + if _, err = server.Listen(context.Background()); err == nil { + t.Fatal("occupied auxiliary STUN listener was accepted") + } + if _, err = os.Stat(configuration.RoomDatabasePath); !os.IsNotExist(err) { + t.Fatalf("STUN bind failure touched room persistence: %v", err) + } + assertPortFree(t, configuration.Port) + assertUDPPortsFree(t, []net.Addr{firstAddress}) + if err = occupied.SetDeadline(time.Now().Add(time.Second)); err != nil { + t.Fatalf("startup rollback closed another listener: %v", err) + } +} + +func assertUDPPortsFree(t *testing.T, addresses []net.Addr) { + t.Helper() + for _, address := range addresses { + listener, err := net.ListenPacket("udp4", address.String()) + if err != nil { + t.Fatalf("STUN listener remained owned: %v", err) + } + _ = listener.Close() + } +} + // --- site access ---------------------------------------------------------- func TestSiteAccessReportsStatusAndIssuesStatelessDayLongCookie(t *testing.T) { @@ -493,6 +587,53 @@ func TestSiteAccessUsesASecureHostCookieForProductionHTTPS(t *testing.T) { } } +func TestLocalPasswordUsesConfiguredDestinationCookieAcrossLANAndPublicLink(t *testing.T) { + configuration, err := config.Local(config.LocalOptions{ + Port: freePort(t), PublicAddress: "192.0.2.10", PublicOrigin: "https://public.example.test", + SiteAccessPassword: testAccessPassword, + }) + if err != nil { + t.Fatal(err) + } + server := start(t, Options{Config: configuration}) + lanOrigin := fmt.Sprintf("http://192.0.2.10:%d", configuration.Port) + for _, origin := range []string{lanOrigin, "https://public.example.test"} { + t.Run(origin, func(t *testing.T) { + destination, _ := url.Parse(origin) + host := func(request *http.Request) { request.Host = destination.Host } + login := server.do(http.MethodPost, "/api/site-access", host, + withOrigin(lanOrigin), withBearer(testAccessPassword)).expectStatus(http.StatusOK) + cookies := (&http.Response{Header: login.header}).Cookies() + if len(cookies) != 1 || cookies[0].Secure != (destination.Scheme == "https") { + t.Fatal("cookie policy does not match its configured destination") + } + jar, _ := cookiejar.New(nil) + jar.SetCookies(destination, cookies) + usable := jar.Cookies(destination) + if len(usable) != 1 { + t.Fatal("browser cookie rules reject the issued login") + } + cookie := usable[0].String() + server.do(http.MethodGet, "/api/site-access", host, withCookie(cookie)). + expect(http.StatusOK, `{"required":true,"authenticated":true}`) + server.do(http.MethodPost, "/api/rooms", host, withOrigin(origin), + withCookie(cookie), withJSON(`{"codeEntryPolicy":"open"}`)).expectStatus(http.StatusCreated) + upgrade := httptest.NewRequest(http.MethodGet, origin+"/signal", nil) + upgrade.Header.Set("Cookie", cookie) + upgrade.Header.Set("X-Forwarded-Proto", "http") + if !server.signalOptions.SiteAccessAtUpgrade(upgrade) { + t.Fatal("WebSocket admission does not use the same cookie policy") + } + if destination.Scheme == "https" { + upgrade.Header.Set("Cookie", strings.TrimPrefix(cookie, "__Host-")) + if server.signalOptions.SiteAccessAtUpgrade(upgrade) { + t.Fatal("public authority accepted the plain LAN cookie name") + } + } + }) + } +} + func TestSiteAccessRequiresAnAllowedOriginAndAnEmptyLoginBody(t *testing.T) { server := start(t, Options{Config: testConfig(t)}) forbidden := `{"error":"Forbidden"}` @@ -1053,94 +1194,15 @@ func TestRestoresStableRoomAuthorityAcrossAnApplicationRestart(t *testing.T) { `{"type":"code-entry-policy-updated","codeEntryPolicy":"open","viewerPasswordEnabled":false}`) } -func TestStartsWithAnInjectedOptionalSFUTokenIssuer(t *testing.T) { - configuration := testConfig(t) - configuration.LiveKit = livekitFallback() - server := start(t, Options{Config: configuration, SFUTokenIssuer: fakeTokenIssuer{}}) - - server.do(http.MethodGet, "/healthz").expect(http.StatusOK, `{"status":"ok"}`) -} - -func TestClearsAStaleManagedLiveKitRoomBeforeServingTraffic(t *testing.T) { - roomControl := sfutest.New() - roomControl.SeedRoom(sfu.ResourceFence{ - RoomID: "42", - ShareGeneration: "stale_share_generation", - PublicationGeneration: "stale_publication_generation", - }, "host", "viewer:stale") - configuration := testConfig(t) - configuration.LiveKit = livekitFallback() - server := start(t, Options{Config: configuration, SFURoomControl: roomControl}) - - if calls := roomControl.InitializeCalls(); calls != 1 { - t.Fatalf("initialize calls = %d, want 1", calls) - } - if deleted := roomControl.StartupDeletedRoomNames(); len(deleted) != 1 { - t.Fatalf("startup deleted rooms = %v, want one", deleted) - } - if count := roomControl.RoomCount(); count != 0 { - t.Fatalf("live rooms = %d, want 0", count) - } - server.do(http.MethodGet, "/healthz").expect(http.StatusOK, `{"status":"ok"}`) -} - -func TestBindsTheApplicationListenerBeforeLiveKitReconciliation(t *testing.T) { - roomControl := sfutest.New() - barrier := make(chan struct{}) - roomControl.SetInitializeBarrier(barrier) - port := freePort(t) - configuration := testConfig(t) - configuration.Port = port - configuration.LiveKit = livekitFallback() - server := newServer(t, Options{Config: configuration, SFURoomControl: roomControl}) - client := testClient(t) - - listening := make(chan error, 1) - go func() { - bound, err := server.Listen(context.Background()) - if err == nil && bound != port { - err = fmt.Errorf("bound port %d, want %d", bound, port) - } - listening <- err - }() - waitFor(t, "LiveKit reconciliation to start", - func() bool { return roomControl.InitializeCalls() == 1 }) - - // Hazard 13: the listener is bound but no traffic is accepted yet. - send(t, client, http.MethodGet, baseURL(port)+"/healthz"). - expect(http.StatusServiceUnavailable, `{"error":"Service starting"}`). - expectHeader("Retry-After", "1"). - expectHeader("Cache-Control", "no-store") - // The /signal upgrade is not served either. - send(t, client, http.MethodGet, baseURL(port)+"/signal", - withOrigin(allowedOrigin), - func(request *http.Request) { - request.Header.Set("Connection", "Upgrade") - request.Header.Set("Upgrade", "websocket") - request.Header.Set("Sec-WebSocket-Version", "13") - request.Header.Set("Sec-WebSocket-Key", "dGhlIHNhbXBsZSBub25jZQ==") - }). - expect(http.StatusServiceUnavailable, `{"error":"Service starting"}`) - - close(barrier) - if err := <-listening; err != nil { - t.Fatalf("Listen: %v", err) - } - send(t, client, http.MethodGet, baseURL(port)+"/healthz"). - expect(http.StatusOK, `{"status":"ok"}`) -} - -func TestMakesNoLiveKitCallWhenAnotherProcessOwnsTheListener(t *testing.T) { +func TestDoesNotOpenDatabaseWhenAnotherProcessOwnsTheListener(t *testing.T) { owner := start(t, Options{Config: testConfig(t)}) ownerPort := ownerBoundPort(t, owner) databasePath := filepath.Join(t.TempDir(), "missing-parent", "rooms.sqlite") - roomControl := sfutest.New() configuration := testConfig(t) configuration.Port = ownerPort configuration.RoomDatabasePath = databasePath - configuration.LiveKit = livekitFallback() - contender := newServer(t, Options{Config: configuration, SFURoomControl: roomControl}) + contender := newServer(t, Options{Config: configuration}) // The TS test asserted the EADDRINUSE code; Windows reports WSAEADDRINUSE, // which errors.Is does not fold into syscall.EADDRINUSE, so the contract @@ -1154,9 +1216,6 @@ func TestMakesNoLiveKitCallWhenAnotherProcessOwnsTheListener(t *testing.T) { if !errors.As(err, &addressError) || addressError.Op != "listen" { t.Fatalf("Listen error = %v, want a listen failure", err) } - if calls := roomControl.InitializeCalls(); calls != 0 { - t.Fatalf("initialize calls = %d, want 0", calls) - } if _, statErr := os.Stat(databasePath); !os.IsNotExist(statErr) { t.Fatalf("the room database was created: %v", statErr) } @@ -1177,30 +1236,25 @@ func ownerBoundPort(t *testing.T, server *harness) int { return port } -func TestRejectsASecondDatabaseOwnerBeforeLiveKitReconciliation(t *testing.T) { +func TestRejectsASecondDatabaseOwnerBeforeServingTraffic(t *testing.T) { databasePath := filepath.Join(t.TempDir(), "rooms.sqlite") ownerConfig := testConfig(t) ownerConfig.RoomDatabasePath = databasePath start(t, Options{Config: ownerConfig}) - roomControl := sfutest.New() contenderConfig := testConfig(t) contenderConfig.Port = freePort(t) contenderConfig.RoomDatabasePath = databasePath - contenderConfig.LiveKit = livekitFallback() - contender := newServer(t, Options{Config: contenderConfig, SFURoomControl: roomControl}) + contender := newServer(t, Options{Config: contenderConfig}) _, err := contender.Listen(context.Background()) if err == nil || !strings.Contains(strings.ToLower(err.Error()), "locked") { t.Fatalf("Listen error = %v, want a locked database", err) } - if calls := roomControl.InitializeCalls(); calls != 0 { - t.Fatalf("initialize calls = %d, want 0", calls) - } assertPortFree(t, contenderConfig.Port) } -func TestRejectsAMismatchedDatabaseBeforeLiveKitReconciliation(t *testing.T) { +func TestRejectsAMismatchedDatabaseBeforeServingTraffic(t *testing.T) { databasePath := filepath.Join(t.TempDir(), "rooms.sqlite") raw, err := sql.Open("sqlite", "file:"+filepath.ToSlash(databasePath)) if err != nil { @@ -1213,119 +1267,35 @@ func TestRejectsAMismatchedDatabaseBeforeLiveKitReconciliation(t *testing.T) { t.Fatalf("close raw database: %v", err) } - roomControl := sfutest.New() configuration := testConfig(t) configuration.Port = freePort(t) configuration.RoomDatabasePath = databasePath - configuration.LiveKit = livekitFallback() - server := newServer(t, Options{Config: configuration, SFURoomControl: roomControl}) + server := newServer(t, Options{Config: configuration}) _, err = server.Listen(context.Background()) if err == nil || !strings.Contains(err.Error(), "Room database application identity does not match") { t.Fatalf("Listen error = %v", err) } - if calls := roomControl.InitializeCalls(); calls != 0 { - t.Fatalf("initialize calls = %d, want 0", calls) - } assertPortFree(t, configuration.Port) } -func TestRejectsAnInaccessibleDatabasePathBeforeLiveKitReconciliation(t *testing.T) { +func TestRejectsAnInaccessibleDatabasePathBeforeServingTraffic(t *testing.T) { databasePath := filepath.Join(t.TempDir(), "missing-parent", "rooms.sqlite") - roomControl := sfutest.New() configuration := testConfig(t) configuration.Port = freePort(t) configuration.RoomDatabasePath = databasePath - configuration.LiveKit = livekitFallback() - server := newServer(t, Options{Config: configuration, SFURoomControl: roomControl}) + server := newServer(t, Options{Config: configuration}) if _, err := server.Listen(context.Background()); err == nil { t.Fatal("Listen accepted an unreachable database path") } - if calls := roomControl.InitializeCalls(); calls != 0 { - t.Fatalf("initialize calls = %d, want 0", calls) - } if _, err := os.Stat(databasePath); !os.IsNotExist(err) { t.Fatalf("the room database was created: %v", err) } assertPortFree(t, configuration.Port) } -func TestHoldsListenerOwnershipUntilAnInFlightStartupSettles(t *testing.T) { - before := runtime.NumGoroutine() - roomControl := sfutest.New() - barrier := make(chan struct{}) - roomControl.SetInitializeBarrier(barrier) - port := freePort(t) - configuration := testConfig(t) - configuration.Port = port - configuration.LiveKit = livekitFallback() - server := newServer(t, Options{Config: configuration, SFURoomControl: roomControl}) - - listening := make(chan int, 1) - go func() { - bound, err := server.Listen(context.Background()) - if err != nil { - bound = -1 - } - listening <- bound - }() - waitFor(t, "LiveKit reconciliation to start", - func() bool { return roomControl.InitializeCalls() == 1 }) - - var closed atomic.Bool - closing := make(chan error, 1) - go func() { - err := server.Close(context.Background()) - closed.Store(true) - closing <- err - }() - time.Sleep(20 * time.Millisecond) - if closed.Load() { - t.Fatal("Close returned while startup was still in flight") - } - // A14: the listener is still owned, so a contender cannot take the port. - if listener, err := net.Listen("tcp", fmt.Sprintf("127.0.0.1:%d", port)); err == nil { - _ = listener.Close() - t.Fatal("the listener was released before startup settled") - } - - close(barrier) - if bound := <-listening; bound != port { - t.Fatalf("Listen returned %d, want %d", bound, port) - } - if err := <-closing; err != nil { - t.Fatalf("Close: %v", err) - } - if !closed.Load() { - t.Fatal("Close never settled") - } - assertPortFree(t, port) - expectGoroutinesSettled(t, before) -} - -func TestClosesTheListenerWhenLiveKitStartupReconciliationFails(t *testing.T) { - before := runtime.NumGoroutine() - roomControl := sfutest.New() - roomControl.SetInitializeError(errors.New("startup reconciliation failed")) - port := freePort(t) - configuration := testConfig(t) - configuration.Port = port - configuration.LiveKit = livekitFallback() - server := newServer(t, Options{Config: configuration, SFURoomControl: roomControl}) - - _, err := server.Listen(context.Background()) - if err == nil || !strings.Contains(err.Error(), "startup reconciliation failed") { - t.Fatalf("Listen error = %v", err) - } - if calls := roomControl.InitializeCalls(); calls != 1 { - t.Fatalf("initialize calls = %d, want 1", calls) - } - assertPortFree(t, port) - expectGoroutinesSettled(t, before) -} - func TestUsesTheConfiguredListenHostByDefault(t *testing.T) { server := start(t, Options{Config: testConfig(t)}) @@ -1335,6 +1305,42 @@ func TestUsesTheConfiguredListenHostByDefault(t *testing.T) { server.do(http.MethodGet, "/healthz").expect(http.StatusOK, `{"status":"ok"}`) } +func TestPreboundListenerHasOneStartupAndShutdownOwner(t *testing.T) { + for _, serve := range []bool{false, true} { + t.Run(fmt.Sprintf("serve=%t", serve), func(t *testing.T) { + listener, err := net.ListenTCP("tcp4", &net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)}) + if err != nil { + t.Fatal(err) + } + address := listener.Addr().(*net.TCPAddr) + server := newServer(t, Options{Listener: listener}) + if serve { + port, err := server.Listen(t.Context()) + if err != nil || port != address.Port || server.listener != listener { + t.Fatalf("startup replaced the prebound listener: port=%d err=%v", port, err) + } + client := &http.Client{Timeout: time.Second} + response, err := client.Get("http://" + address.String() + "/healthz") + if err != nil { + t.Fatal(err) + } + _ = response.Body.Close() + if response.StatusCode != http.StatusOK { + t.Fatalf("prebound server health = %d", response.StatusCode) + } + } + if err := server.Close(t.Context()); err != nil { + t.Fatal(err) + } + rebound, err := net.ListenTCP("tcp4", address) + if err != nil { + t.Fatalf("prebound listener survived shutdown: %v", err) + } + _ = rebound.Close() + }) + } +} + func TestReportsProcessLivenessWithoutAccessChecks(t *testing.T) { server := start(t, Options{Config: testConfig(t)}) @@ -1379,8 +1385,8 @@ func TestListenIsSingleShotAndRefusedAfterClose(t *testing.T) { // --- request failures and server settings --------------------------------- // The createServer .catch of app.ts: a handler that throws answers 500 and the -// record carries the method and the path only (D11). -func TestRequestFailureAnswers500AndLogsMethodAndPathOnly(t *testing.T) { +// record carries fixed method and route categories without request data. +func TestRequestFailureAnswers500AndLogsRequestCategoriesOnly(t *testing.T) { var logged bytes.Buffer server := newServer(t, Options{ Config: testConfig(t), @@ -1393,7 +1399,7 @@ func TestRequestFailureAnswers500AndLogsMethodAndPathOnly(t *testing.T) { recorder := httptest.NewRecorder() server.ServeHTTP(recorder, - httptest.NewRequest(http.MethodGet, "/boom?grant=viewer-grant-that-must-not-be-logged", nil)) + httptest.NewRequest(http.MethodGet, "/viewer-grant-that-must-not-be-logged?grant=viewer-grant-that-must-not-be-logged", nil)) if recorder.Code != http.StatusInternalServerError { t.Fatalf("status = %d, want 500", recorder.Code) @@ -1404,12 +1410,18 @@ func TestRequestFailureAnswers500AndLogsMethodAndPathOnly(t *testing.T) { record := logged.String() if !strings.Contains(record, "HTTP request failed") || !strings.Contains(record, "method=GET") || - !strings.Contains(record, "path=/boom") { + !strings.Contains(record, "route=frontend") || strings.Contains(record, "path=") { t.Fatalf("log record = %q", record) } if strings.Contains(record, "viewer-grant-that-must-not-be-logged") { t.Fatalf("the failure log leaked the request: %q", record) } + logged.Reset() + server.ServeHTTP(httptest.NewRecorder(), + httptest.NewRequest("private-method-token", "/private-path-token", nil)) + if record := logged.String(); !strings.Contains(record, "method=other") || strings.Contains(record, "private-") { + t.Fatalf("the failure log leaked a custom method: %q", record) + } } // A failure after the response started destroyed the socket in Node; in Go the @@ -1495,46 +1507,10 @@ func TestSignalUpgradeIsDispatchedToTheSignalingServer(t *testing.T) { // panickingRoomControl is a reconciliation step that fails with a panic rather // than an error, which is what a nil dereference inside a future startup step // would look like. -type panickingRoomControl struct{ sfu.RoomControl } - -func (panickingRoomControl) Initialize(context.Context) error { - panic(errors.New("startup reconciliation panicked")) -} // app.ts:219 awaited startupOperation inside a try/catch, so a rejected // startup still settled the promise shutdown waited on. A panic out of start() // must release the same latch. -func TestShutdownIsNotBlockedByAStartupThatPanicked(t *testing.T) { - configuration := testConfig(t) - configuration.LiveKit = livekitFallback() - server, err := New(Options{ - Config: configuration, - SFURoomControl: panickingRoomControl{}, - SFUTokenIssuer: fakeTokenIssuer{}, - HeartbeatIntervalMs: 60_000, - CleanupIntervalMs: 60_000, - }) - if err != nil { - t.Fatalf("New: %v", err) - } - - func() { - defer func() { - if recovered := recover(); recovered == nil { - t.Fatal("Listen swallowed the startup panic") - } - }() - _, _ = server.Listen(context.Background()) - }() - - closed := make(chan error, 1) - go func() { closed <- server.Close(context.Background()) }() - select { - case <-closed: - case <-time.After(5 * time.Second): - t.Fatal("Close blocked on the startup latch after a panicking startup") - } -} // syncBuffer collects log records written from the Serve goroutine. type syncBuffer struct { @@ -1574,4 +1550,7 @@ func TestReportsAListenerThatDiesWhileServing(t *testing.T) { waitFor(t, "the serve failure to be reported", func() bool { return strings.Contains(logged.String(), "Screener HTTP server stopped unexpectedly") }) + if record := logged.String(); !strings.Contains(record, "errorType=*net.OpError") || strings.Contains(record, "error=") || strings.Contains(record, "127.0.0.1") { + t.Fatalf("listener failure must log only the error type: %q", record) + } } diff --git a/internal/server/app/routes.go b/internal/server/app/routes.go index d1a7372..4ed72c1 100644 --- a/internal/server/app/routes.go +++ b/internal/server/app/routes.go @@ -33,8 +33,7 @@ func (s *Server) ServeHTTP(writer http.ResponseWriter, request *http.Request) { if recovered == http.ErrAbortHandler { panic(recovered) } - // TS: console.error("HTTP request failed", {method, path, error}). - // D11 keeps the method and the path only. + // Persist only fixed request categories, never the panic or raw URL. s.requestFailed(recorder, request) }() @@ -143,16 +142,17 @@ func allowMethod(writer http.ResponseWriter, request *http.Request, allowed stri // handleSiteAccess is handleSiteAccessRequest of app.ts. func (s *Server) handleSiteAccess(writer http.ResponseWriter, request *http.Request) { noStoreJSON(writer) + access := s.siteAccessForRequest(request) if request.Method == http.MethodGet { status := siteAccessBody{ - Required: s.siteAccess.required(), - Authenticated: s.siteAccess.isAuthenticated(cookieHeader(request)), + Required: access.required(), + Authenticated: access.isAuthenticated(cookieHeader(request)), } if status.Required && status.Authenticated { // The Set-Cookie value is written verbatim, in the TS attribute // order (D11). - writer.Header().Set("Set-Cookie", s.siteAccess.createCookie()) + writer.Header().Set("Set-Cookie", access.createCookie()) } sendJSON(writer, http.StatusOK, status) return @@ -172,16 +172,16 @@ func (s *Server) handleSiteAccess(writer http.ResponseWriter, request *http.Requ sendJSON(writer, http.StatusBadRequest, errorBody{"Request body is not accepted"}) return } - if !s.siteAccess.required() { + if !access.required() { sendJSON(writer, http.StatusOK, siteAccessBody{Required: false, Authenticated: true}) return } - if provided := bearerToken(request); provided == "" || !s.siteAccess.passwordMatches(provided) { + if provided := bearerToken(request); provided == "" || !access.passwordMatches(provided) { writer.Header().Set("WWW-Authenticate", "Bearer") sendJSON(writer, http.StatusUnauthorized, errorBody{"Unauthorized"}) return } - writer.Header().Set("Set-Cookie", s.siteAccess.createCookie()) + writer.Header().Set("Set-Cookie", access.createCookie()) sendJSON(writer, http.StatusOK, siteAccessBody{Required: true, Authenticated: true}) } @@ -360,7 +360,7 @@ func (s *Server) allowedRequestOrigin(request *http.Request) bool { // roomCreationAuthorized ports isRoomCreationAuthorized. func (s *Server) roomCreationAuthorized(request *http.Request) bool { - return !s.siteAccess.required() || s.siteAccess.isAuthenticated(cookieHeader(request)) + return s.siteAccessForRequest(request).isAuthenticated(cookieHeader(request)) } // createRoomResponse of app.ts. signal.InviteURL is the one owner of the invite @@ -378,12 +378,27 @@ func (s *Server) createRoomResponse(created room.CreatedRoom) protocol.CreateRoo } } -// requestFailed is the createServer .catch: log the method and the path only — -// never a body, header, token or grant (D11) — then answer 500 unless the -// response already started, where Node destroyed the socket. +// requestFailed logs fixed request categories, then answers 500 or aborts a +// response that already started. Request paths and arbitrary methods are private. func (s *Server) requestFailed(writer http.ResponseWriter, request *http.Request) { - s.logger.Error("HTTP request failed", - "method", request.Method, "path", requestPath(request)) + method := request.Method + switch method { + case http.MethodGet, http.MethodPost, http.MethodHead, http.MethodOptions, + http.MethodPut, http.MethodPatch, http.MethodDelete, http.MethodConnect, http.MethodTrace: + default: + method = "other" + } + path := requestPath(request) + category := "frontend" + switch { + case isUpgradeRequest(request): + category = "signaling" + case path == "/healthz": + category = "health" + case strings.HasPrefix(path, "/api/"): + category = "api" + } + s.logger.Error("HTTP request failed", "method", method, "route", category) if recorder, ok := writer.(*responseRecorder); ok && recorder.wrote { panic(http.ErrAbortHandler) } diff --git a/internal/server/config/config.go b/internal/server/config/config.go index 8f38a2c..f6a5ba8 100644 --- a/internal/server/config/config.go +++ b/internal/server/config/config.go @@ -8,6 +8,7 @@ import ( "errors" "fmt" "math" + "net/netip" "net/url" "path/filepath" "regexp" @@ -31,7 +32,6 @@ const ( const ( minSiteAccessPasswordBytes = 8 maxSiteAccessPasswordBytes = 128 - minLiveKitAPISecretBytes = 32 defaultMaxViewersPerRoom = 8 defaultRoomLeaseSeconds = 86_400 defaultPort = 8787 @@ -67,18 +67,16 @@ var removedEnvironmentVariables = []struct{ name, reason string }{ {"NODE_ENV", "use SCREENER_ENV"}, } -// LiveKitFallback ports LiveKitFallbackConfig. URL and APIURL are serialised -// WHATWG origins. -type LiveKitFallback struct { - URL string - APIURL string - APIKey string - APISecret string +// SFUConfig enables the embedded UDP media listener. +type SFUConfig struct { + ListenHost string + Port int + PublicIP string } // Config ports ServerConfig. Fields the TypeScript left `undefined` are the // zero value here: an empty SiteAccessPassword means site access is open, an -// empty RoomDatabasePath means memory mode, and a nil LiveKit means no SFU +// empty RoomDatabasePath means memory mode, and a nil SFU means no SFU // fallback. type Config struct { Env Environment @@ -93,9 +91,12 @@ type Config struct { RoomLeaseMs int64 MaxViewersPerRoom int EndpointMediaCopyCapacity int - LiveKit *LiveKitFallback + SFU *SFUConfig STUNURLs []string - NATPredictionEnabled bool + // STUNListenAddresses is populated only by Hosted configuration. Local + // discovery URLs do not make the Client an externally reachable STUN server. + STUNListenAddresses []string + NATPredictionEnabled bool // NATPredictionSTUNURLs is nil where the TypeScript left it `undefined`, // which IceConfig distinguishes from a configured empty list. NATPredictionSTUNURLs []string @@ -168,6 +169,13 @@ func Load(env map[string]string) (Config, error) { "STUN_URLS must contain at most %d URLs when NAT_PREDICTION_ENABLED=true", protocol.MaxIceServerURLs-protocol.MaxNatPredictionAuxiliaryStunURLs) } + var stunListeners []string + if len(stunURLs) > 0 { + stunListeners, err = stunListenAddresses(env["STUN_LISTEN_HOST"], natPredictionEnabled) + if err != nil { + return Config{}, err + } + } maxViewersPerRoom, err := parseBoundedInteger(env["MAX_VIEWERS_PER_ROOM"], defaultMaxViewersPerRoom, "MAX_VIEWERS_PER_ROOM", 1, protocol.MaxViewersPerRoomLimit) if err != nil { @@ -179,26 +187,11 @@ func Load(env map[string]string) (Config, error) { if err != nil { return Config{}, err } - liveKit, err := parseLiveKitFallback(env, environment) + sfu, err := parseSFU(env) if err != nil { return Config{}, err } - secrets := make([]string, 0, 3) - if siteAccessPassword != "" { - secrets = append(secrets, siteAccessPassword) - } - if liveKit != nil { - secrets = append(secrets, liveKit.APIKey, liveKit.APISecret) - } - distinct := make(map[string]struct{}, len(secrets)) - for _, secret := range secrets { - distinct[secret] = struct{}{} - } - if len(distinct) != len(secrets) { - return Config{}, errors.New( - "SITE_ACCESS_PASSWORD, LIVEKIT_API_KEY, and LIVEKIT_API_SECRET must use independent values") - } // Buffer.byteLength is the UTF-8 length, which is len() in Go. The pattern // already restricts the value to one byte per character. if siteAccessPassword != "" && @@ -238,8 +231,9 @@ func Load(env map[string]string) (Config, error) { RoomLeaseMs: roomLeaseSeconds * 1_000, MaxViewersPerRoom: int(maxViewersPerRoom), EndpointMediaCopyCapacity: int(endpointMediaCopyCapacity), - LiveKit: liveKit, + SFU: sfu, STUNURLs: stunURLs, + STUNListenAddresses: stunListeners, NATPredictionEnabled: natPredictionEnabled, }, nil } @@ -309,60 +303,35 @@ func parseRoomDatabasePath(value string) (string, error) { return path, nil } -// parseLiveKitFallback ports parseLiveKitFallback. -func parseLiveKitFallback(env map[string]string, environment Environment) (*LiveKitFallback, error) { - rawURL := strings.TrimFunc(env["LIVEKIT_URL"], protocol.IsJSWhitespace) - rawAPIURL := strings.TrimFunc(env["LIVEKIT_API_URL"], protocol.IsJSWhitespace) - apiKey := strings.TrimFunc(env["LIVEKIT_API_KEY"], protocol.IsJSWhitespace) - apiSecret := strings.TrimFunc(env["LIVEKIT_API_SECRET"], protocol.IsJSWhitespace) - - configured := 0 - for _, value := range []string{rawURL, rawAPIURL, apiKey, apiSecret} { - if value != "" { - configured++ - } - } - if configured == 0 { +func parseSFU(env map[string]string) (*SFUConfig, error) { + rawPort := strings.TrimSpace(env["SFU_UDP_PORT"]) + if rawPort == "" { return nil, nil } - if configured != 4 { - return nil, errors.New( - "LIVEKIT_URL, LIVEKIT_API_URL, LIVEKIT_API_KEY, and LIVEKIT_API_SECRET must be configured together") - } - if len(apiSecret) < minLiveKitAPISecretBytes { - return nil, fmt.Errorf("LIVEKIT_API_SECRET must contain at least %d bytes", minLiveKitAPISecretBytes) - } - - parsedURL, err := parseOriginURL(rawURL, "LIVEKIT_URL", "ws", "wss") + port, err := parseBoundedInteger(rawPort, 0, "SFU_UDP_PORT", 1, maxPort) if err != nil { return nil, err } - if environment == EnvironmentProduction && parsedURL.Scheme != "wss" { - return nil, errors.New("LIVEKIT_URL must use wss in production") + listenHost := strings.TrimSpace(env["SFU_LISTEN_HOST"]) + if listenHost == "" { + listenHost = "0.0.0.0" } - parsedAPIURL, err := parseOriginURL(rawAPIURL, "LIVEKIT_API_URL", "http", "https") - if err != nil { - return nil, err + if address, err := netip.ParseAddr(listenHost); err != nil || !address.Is4() { + return nil, errors.New("SFU_LISTEN_HOST must be an IPv4 address") } - if environment == EnvironmentProduction && parsedAPIURL.Scheme != "https" && - !isLoopbackHostname(parsedAPIURL.Hostname()) { - return nil, errors.New("LIVEKIT_API_URL must use https or loopback in production") + publicIP := strings.TrimSpace(env["SFU_PUBLIC_IP"]) + if publicIP != "" { + if address, err := netip.ParseAddr(publicIP); err != nil || !address.Is4() { + return nil, errors.New("SFU_PUBLIC_IP must be an IPv4 address") + } } - return &LiveKitFallback{ - URL: Origin(parsedURL), - APIURL: Origin(parsedAPIURL), - APIKey: apiKey, - APISecret: apiSecret, + return &SFUConfig{ + ListenHost: listenHost, + Port: int(port), + PublicIP: publicIP, }, nil } -// isLoopbackHostname ports isLoopbackHostname. url.URL.Hostname strips the IPv6 -// brackets, so only the unbracketed "::1" can actually match here. -func isLoopbackHostname(hostname string) bool { - return hostname == "localhost" || hostname == "127.0.0.1" || - hostname == "[::1]" || hostname == "::1" -} - // parseURLList ports parseUrlList. func parseURLList(value string, name string) ([]string, error) { if strings.TrimFunc(value, protocol.IsJSWhitespace) == "" { @@ -431,9 +400,8 @@ func toOrigin(value string) (string, error) { return Origin(parsed), nil } -// parseOriginURL ports the `new URL` + protocol + shape ladder that -// PUBLIC_BASE_URL, LIVEKIT_URL and LIVEKIT_API_URL each repeat, and returns the -// WHATWG-normalised URL (lowercase host, default port removed, path "/"). +// parseOriginURL validates an origin and returns the WHATWG-normalised URL +// (lowercase host, default port removed, path "/"). func parseOriginURL(value, name, scheme, alternative string) (*url.URL, error) { parsed, err := url.Parse(value) // net/url accepts relative references and opaque URLs that the WHATWG diff --git a/internal/server/config/config_test.go b/internal/server/config/config_test.go index ee6ebf1..8adfeff 100644 --- a/internal/server/config/config_test.go +++ b/internal/server/config/config_test.go @@ -24,16 +24,12 @@ func env(parts ...map[string]string) map[string]string { return merged } -var ( - liveKitAdmission = map[string]string{"LIVEKIT_API_URL": "https://livekit-api.test"} - liveKitSecret = strings.Repeat("s", 32) - productionBase = map[string]string{ - "SCREENER_ENV": "production", - "PUBLIC_BASE_URL": "https://share.test", - "SITE_ACCESS_PASSWORD": "host-password-12", - "STUN_URLS": "stun:stun.test:3478", - } -) +var productionBase = map[string]string{ + "SCREENER_ENV": "production", + "PUBLIC_BASE_URL": "https://share.test", + "SITE_ACCESS_PASSWORD": "host-password-12", + "STUN_URLS": "stun:stun.test:3478", +} func stunList(count int, prefix string) string { urls := make([]string, 0, count) @@ -74,6 +70,9 @@ func TestLoadDevelopmentDefaults(t *testing.T) { if len(config.STUNURLs) != 0 { t.Errorf("STUNURLs = %v", config.STUNURLs) } + if len(config.STUNListenAddresses) != 0 { + t.Errorf("STUNListenAddresses = %v", config.STUNListenAddresses) + } if config.NATPredictionEnabled { t.Error("NATPredictionEnabled = true") } @@ -86,8 +85,8 @@ func TestLoadDevelopmentDefaults(t *testing.T) { if config.EndpointMediaCopyCapacity != 2 { t.Errorf("EndpointMediaCopyCapacity = %d", config.EndpointMediaCopyCapacity) } - if config.LiveKit != nil { - t.Errorf("LiveKit = %+v", config.LiveKit) + if config.SFU != nil { + t.Errorf("SFU = %+v", config.SFU) } if config.RoomLeaseMs != 86_400_000 { t.Errorf("RoomLeaseMs = %d", config.RoomLeaseMs) @@ -97,21 +96,42 @@ func TestLoadDevelopmentDefaults(t *testing.T) { } } -func TestLoadLiveKitTuple(t *testing.T) { +func TestLoadSFU(t *testing.T) { config := mustLoad(t, map[string]string{ - "LIVEKIT_URL": " ws://livekit.test:7880 ", - "LIVEKIT_API_URL": " http://livekit.test:7880 ", - "LIVEKIT_API_KEY": " test-key ", - "LIVEKIT_API_SECRET": " " + liveKitSecret + " ", + "LISTEN_HOST": "127.0.0.1", + "SFU_UDP_PORT": " 7882 ", + "SFU_LISTEN_HOST": " 192.0.2.5 ", + "SFU_PUBLIC_IP": " 198.51.100.5 ", }) - want := LiveKitFallback{ - URL: "ws://livekit.test:7880", - APIURL: "http://livekit.test:7880", - APIKey: "test-key", - APISecret: liveKitSecret, + want := SFUConfig{ + ListenHost: "192.0.2.5", + Port: 7882, + PublicIP: "198.51.100.5", } - if config.LiveKit == nil || *config.LiveKit != want { - t.Errorf("LiveKit = %+v, want %+v", config.LiveKit, want) + if config.SFU == nil || *config.SFU != want { + t.Errorf("SFU = %+v, want %+v", config.SFU, want) + } +} + +func TestLoadSFUOptional(t *testing.T) { + for _, values := range []map[string]string{ + nil, + {"SFU_UDP_PORT": ""}, + {"SFU_UDP_PORT": " "}, + {"SFU_LISTEN_HOST": "192.0.2.5", "SFU_PUBLIC_IP": "198.51.100.5"}, + } { + if config := mustLoad(t, values); config.SFU != nil { + t.Errorf("SFU = %+v without an enabled port", config.SFU) + } + } + for _, port := range []int{1, 7882, maxPort} { + config := mustLoad(t, env(productionBase, map[string]string{ + "SFU_UDP_PORT": strconv.Itoa(port), + })) + want := SFUConfig{ListenHost: "0.0.0.0", Port: port} + if config.SFU == nil || *config.SFU != want { + t.Errorf("SFU = %+v, want %+v", config.SFU, want) + } } } @@ -176,6 +196,19 @@ func TestLoadAccepts(t *testing.T) { if !slices.Equal(c.STUNURLs, []string{"stun:stun.test:3478"}) { t.Errorf("STUNURLs = %v", c.STUNURLs) } + if !slices.Equal(c.STUNListenAddresses, []string{"0.0.0.0:3478"}) { + t.Errorf("STUNListenAddresses = %v", c.STUNListenAddresses) + } + }}, + {"independent STUN bind host", env(productionBase, map[string]string{ + "LISTEN_HOST": "127.0.0.1", "STUN_LISTEN_HOST": " 192.0.2.5 ", + "NAT_PREDICTION_ENABLED": "true", + }), func(t *testing.T, c Config) { + if !slices.Equal(c.STUNListenAddresses, []string{ + "192.0.2.5:3478", "192.0.2.5:3479", "192.0.2.5:3480", + }) || !slices.Equal(c.STUNURLs, []string{"stun:stun.test:3478"}) { + t.Fatalf("binding and advertisement were coupled: %+v", c) + } }}, {"production site access password", env(productionBase, map[string]string{"SITE_ACCESS_PASSWORD": "easy-key"}), @@ -184,25 +217,6 @@ func TestLoadAccepts(t *testing.T) { t.Errorf("SiteAccessPassword = %q", c.SiteAccessPassword) } }}, - {"STUN-only production with LiveKit", env(productionBase, liveKitAdmission, map[string]string{ - "LIVEKIT_URL": "wss://livekit.test", - "LIVEKIT_API_KEY": "test-key", - "LIVEKIT_API_SECRET": liveKitSecret, - }), func(t *testing.T, c Config) { - if c.LiveKit == nil || c.LiveKit.URL != "wss://livekit.test" { - t.Errorf("LiveKit = %+v", c.LiveKit) - } - }}, - {"production loopback LiveKit control", env(productionBase, map[string]string{ - "LIVEKIT_URL": "wss://livekit.test", - "LIVEKIT_API_URL": "http://127.0.0.1:7880", - "LIVEKIT_API_KEY": "test-key", - "LIVEKIT_API_SECRET": liveKitSecret, - }), func(t *testing.T, c Config) { - if c.LiveKit == nil || c.LiveKit.APIURL != "http://127.0.0.1:7880" { - t.Errorf("LiveKit = %+v", c.LiveKit) - } - }}, {"uppercase STUN scheme and IPv6 host", map[string]string{"STUN_URLS": "STUN:[2001:db8::1]:3478"}, func(t *testing.T, c Config) { if !slices.Equal(c.STUNURLs, []string{"STUN:[2001:db8::1]:3478"}) { @@ -289,59 +303,28 @@ func TestLoadRejects(t *testing.T) { env map[string]string want string }{ - // Partial LiveKit credential tuples. - {"only LiveKit URL", map[string]string{"LIVEKIT_URL": "wss://livekit.test"}, - "LIVEKIT_URL, LIVEKIT_API_URL, LIVEKIT_API_KEY, and LIVEKIT_API_SECRET must be configured together"}, - {"only LiveKit control URL", liveKitAdmission, - "LIVEKIT_URL, LIVEKIT_API_URL, LIVEKIT_API_KEY, and LIVEKIT_API_SECRET must be configured together"}, - {"only LiveKit key", map[string]string{"LIVEKIT_API_KEY": "test-key"}, - "must be configured together"}, - {"only LiveKit secret", map[string]string{"LIVEKIT_API_SECRET": liveKitSecret}, - "must be configured together"}, - {"LiveKit without secret", env(liveKitAdmission, map[string]string{ - "LIVEKIT_URL": "wss://livekit.test", "LIVEKIT_API_KEY": "test-key", - }), "must be configured together"}, - - // Invalid LiveKit media origins. - {"LiveKit https scheme", liveKitURL("https://livekit.test"), "LIVEKIT_URL must use ws or wss"}, - {"LiveKit credentials", liveKitURL("wss://user:pass@livekit.test"), - "LIVEKIT_URL must be an origin without credentials, path, query, or fragment"}, - {"LiveKit path", liveKitURL("wss://livekit.test/rtc"), "LIVEKIT_URL must be an origin"}, - {"LiveKit query", liveKitURL("wss://livekit.test?token=value"), "LIVEKIT_URL must be an origin"}, - {"LiveKit fragment", liveKitURL("wss://livekit.test#fragment"), "LIVEKIT_URL must be an origin"}, - {"LiveKit not a URL", liveKitURL("livekit.test"), "LIVEKIT_URL must be a valid ws or wss origin"}, - {"LiveKit ws in production", env(productionBase, liveKitAdmission, map[string]string{ - "LIVEKIT_URL": "ws://livekit.test:7880", "LIVEKIT_API_KEY": "test-key", - "LIVEKIT_API_SECRET": liveKitSecret, - }), "LIVEKIT_URL must use wss in production"}, - - // Invalid LiveKit control origins. - {"control ws scheme", liveKitAPIURL("ws://livekit-api.test"), "LIVEKIT_API_URL must use http or https"}, - {"control credentials", liveKitAPIURL("http://user:pass@livekit-api.test"), - "LIVEKIT_API_URL must be an origin without credentials, path, query, or fragment"}, - {"control path", liveKitAPIURL("http://livekit-api.test/rtc"), "LIVEKIT_API_URL must be an origin"}, - {"control query", liveKitAPIURL("http://livekit-api.test?token=value"), "LIVEKIT_API_URL must be an origin"}, - {"control fragment", liveKitAPIURL("http://livekit-api.test#fragment"), "LIVEKIT_API_URL must be an origin"}, - {"control not a URL", liveKitAPIURL("livekit-api.test"), - "LIVEKIT_API_URL must be a valid http or https origin"}, - {"control plaintext in production", env(productionBase, map[string]string{ - "LIVEKIT_URL": "wss://livekit.test", "LIVEKIT_API_URL": "http://livekit-api.test:7880", - "LIVEKIT_API_KEY": "test-key", "LIVEKIT_API_SECRET": liveKitSecret, - }), "must use https or loopback"}, - {"short LiveKit secret", env(liveKitAdmission, map[string]string{ - "LIVEKIT_URL": "wss://livekit.test", "LIVEKIT_API_KEY": "test-key", - "LIVEKIT_API_SECRET": "too-short", - }), "LIVEKIT_API_SECRET must contain at least 32 bytes"}, - - // Reused infrastructure secrets. - {"site password reused as LiveKit secret", env(liveKitAdmission, map[string]string{ - "SITE_ACCESS_PASSWORD": strings.Repeat("x", 32), "LIVEKIT_URL": "wss://livekit.test", - "LIVEKIT_API_KEY": "test-key", "LIVEKIT_API_SECRET": strings.Repeat("x", 32), - }), "must use independent values"}, - {"LiveKit key reused as LiveKit secret", env(liveKitAdmission, map[string]string{ - "LIVEKIT_URL": "wss://livekit.test", "LIVEKIT_API_KEY": strings.Repeat("x", 32), - "LIVEKIT_API_SECRET": strings.Repeat("x", 32), - }), "must use independent values"}, + {"SFU port zero", map[string]string{"SFU_UDP_PORT": "0"}, + "SFU_UDP_PORT must be a positive integer"}, + {"SFU port negative", map[string]string{"SFU_UDP_PORT": "-1"}, + "SFU_UDP_PORT must be a positive integer"}, + {"SFU port fractional", map[string]string{"SFU_UDP_PORT": "7882.5"}, + "SFU_UDP_PORT must be a positive integer"}, + {"SFU port not a number", map[string]string{"SFU_UDP_PORT": "media"}, + "SFU_UDP_PORT must be a positive integer"}, + {"SFU port above UDP ceiling", map[string]string{"SFU_UDP_PORT": "65536"}, + "SFU_UDP_PORT must be between 1 and 65535"}, + {"SFU bind hostname", map[string]string{"SFU_UDP_PORT": "7882", "SFU_LISTEN_HOST": "sfu.test"}, + "SFU_LISTEN_HOST must be an IPv4 address"}, + {"SFU bind IPv6", map[string]string{"SFU_UDP_PORT": "7882", "SFU_LISTEN_HOST": "::1"}, + "SFU_LISTEN_HOST must be an IPv4 address"}, + {"SFU bind mapped IPv6", map[string]string{"SFU_UDP_PORT": "7882", "SFU_LISTEN_HOST": "::ffff:192.0.2.5"}, + "SFU_LISTEN_HOST must be an IPv4 address"}, + {"SFU public hostname", map[string]string{"SFU_UDP_PORT": "7882", "SFU_PUBLIC_IP": "sfu.test"}, + "SFU_PUBLIC_IP must be an IPv4 address"}, + {"SFU public IPv6", map[string]string{"SFU_UDP_PORT": "7882", "SFU_PUBLIC_IP": "2001:db8::5"}, + "SFU_PUBLIC_IP must be an IPv4 address"}, + {"SFU public mapped IPv6", map[string]string{"SFU_UDP_PORT": "7882", "SFU_PUBLIC_IP": "::ffff:198.51.100.5"}, + "SFU_PUBLIC_IP must be an IPv4 address"}, // Bounded integers. {"viewer limit above ceiling", @@ -411,6 +394,12 @@ func TestLoadRejects(t *testing.T) { // STUN lists. {"STUN path", map[string]string{"STUN_URLS": "stun:stun.test/path"}, "STUN_URLS contains an invalid STUN URL"}, + {"STUN hostname is not a local bind address", map[string]string{ + "STUN_URLS": "stun:stun.test:3478", "STUN_LISTEN_HOST": "stun.test", + }, "STUN_LISTEN_HOST must be an IPv4 address"}, + {"STUN IPv6 listener is unsupported", map[string]string{ + "STUN_URLS": "stun:stun.test:3478", "STUN_LISTEN_HOST": "::1", + }, "STUN_LISTEN_HOST must be an IPv4 address"}, {"STUN query", map[string]string{"STUN_URLS": "stun:stun.test?transport=udp"}, "STUN_URLS contains an invalid STUN URL"}, {"STUN fragment", map[string]string{"STUN_URLS": "stun:stun.test#fragment"}, @@ -523,8 +512,6 @@ func TestOrigin(t *testing.T) { {"http://Example.com:80", "http", "https", "http://example.com"}, {"https://share.test:0443", "http", "https", "https://share.test"}, {"http://localhost:9123", "http", "https", "http://localhost:9123"}, - {"wss://LiveKit.test:443", "ws", "wss", "wss://livekit.test"}, - {"ws://livekit.test:7880", "ws", "wss", "ws://livekit.test:7880"}, {"https://[2001:DB8::1]:8443", "http", "https", "https://[2001:db8::1]:8443"}, {"http://[::1]:7880", "http", "https", "http://[::1]:7880"}, } @@ -543,16 +530,3 @@ func TestOrigin(t *testing.T) { }) } } - -func liveKitURL(value string) map[string]string { - return env(liveKitAdmission, map[string]string{ - "LIVEKIT_URL": value, "LIVEKIT_API_KEY": "test-key", "LIVEKIT_API_SECRET": liveKitSecret, - }) -} - -func liveKitAPIURL(value string) map[string]string { - return map[string]string{ - "LIVEKIT_URL": "wss://livekit.test", "LIVEKIT_API_URL": value, - "LIVEKIT_API_KEY": "test-key", "LIVEKIT_API_SECRET": liveKitSecret, - } -} diff --git a/internal/server/config/ice.go b/internal/server/config/ice.go index aafa831..617e038 100644 --- a/internal/server/config/ice.go +++ b/internal/server/config/ice.go @@ -2,6 +2,7 @@ package config import ( "fmt" + "net" "net/url" "strconv" "strings" @@ -17,6 +18,24 @@ const natPredictionBasePort = 3478 // answers reveal the peer's port-allocation stride. var natPredictionAuxiliaryPorts = [...]int{natPredictionBasePort + 1, natPredictionBasePort + 2} +func stunListenAddresses(host string, prediction bool) ([]string, error) { + host = strings.TrimSpace(host) + if host == "" { + host = "0.0.0.0" + } + address := net.ParseIP(host) + if address == nil || address.To4() == nil || strings.ContainsRune(host, ':') { + return nil, fmt.Errorf("STUN_LISTEN_HOST must be an IPv4 address") + } + listeners := []string{net.JoinHostPort(host, strconv.Itoa(natPredictionBasePort))} + if prediction { + for _, port := range natPredictionAuxiliaryPorts { + listeners = append(listeners, net.JoinHostPort(host, strconv.Itoa(port))) + } + } + return listeners, nil +} + // NATPredictionStunURLs ports natPredictionStunUrls: the two auxiliary // listeners derived from the first ordinary STUN authority on UDP 3478. func NATPredictionStunURLs(stunURLs []string) []string { diff --git a/internal/server/config/local_test.go b/internal/server/config/local_test.go index 6c7251d..e6d9b9e 100644 --- a/internal/server/config/local_test.go +++ b/internal/server/config/local_test.go @@ -52,6 +52,9 @@ func TestLocalMapsClientInputs(t *testing.T) { if len(config.STUNURLs) != 0 { t.Errorf("STUNURLs = %v", config.STUNURLs) } + if len(config.STUNListenAddresses) != 0 { + t.Fatalf("Local created STUN listeners: %v", config.STUNListenAddresses) + } if config.NATPredictionEnabled { t.Error("NATPredictionEnabled = true") } @@ -67,8 +70,8 @@ func TestLocalMapsClientInputs(t *testing.T) { if !reflect.DeepEqual(config.AllowedOrigins, want) { t.Errorf("AllowedOrigins = %v, want %v", config.AllowedOrigins, want) } - if config.LiveKit != nil { - t.Errorf("LiveKit = %+v", config.LiveKit) + if config.SFU != nil { + t.Errorf("SFU = %+v", config.SFU) } if config.RoomDatabasePath != "" { t.Errorf("RoomDatabasePath = %q", config.RoomDatabasePath) @@ -105,6 +108,12 @@ func TestLocalTunnelledClient(t *testing.T) { if !config.NATPredictionEnabled { t.Error("NATPredictionEnabled = false") } + if len(config.STUNListenAddresses) != 0 { + t.Fatalf("public discovery created Local STUN listeners: %v", config.STUNListenAddresses) + } + if config.SFU != nil { + t.Errorf("public link created an SFU listener: %+v", config.SFU) + } if !slices.Equal(config.NATPredictionSTUNURLs, []string{"stun:survey-a.example:3478", "stun:survey-b.example:3478"}) { t.Errorf("NATPredictionSTUNURLs = %v", config.NATPredictionSTUNURLs) diff --git a/internal/server/protocol/client.go b/internal/server/protocol/client.go index b53f307..aa4c73c 100644 --- a/internal/server/protocol/client.go +++ b/internal/server/protocol/client.go @@ -228,6 +228,8 @@ func DecodeClientMessage(data []byte) (ClientMessage, error) { return decodeRouteFailed(data) case "refresh-sfu": return decodeRefreshSfu(data) + case "sfu-signal": + return decodeSfuSignal(data) case "request-route-diagnostic": return decodeEmptyClientMessage(data, RequestRouteDiagnosticMessage{Type: messageType}) case "viewer-quality-evidence": diff --git a/internal/server/protocol/constants.go b/internal/server/protocol/constants.go index 9315642..6c75d64 100644 --- a/internal/server/protocol/constants.go +++ b/internal/server/protocol/constants.go @@ -13,10 +13,9 @@ const ( MaxViewersPerRoomLimit = 20 MaxParticipantsPerRoomLimit = MaxViewersPerRoomLimit + 1 MaxSignalBytes = 64 * 1024 - SignalingProtocol = "screener-v21" + SignalingProtocol = "screener-v22" RoomCodeLength = 4 MaxMediaRouteRevision = MaxSafeInteger - MaxSfuTokenLength = 8 * 1024 MaxIceServerURLs = 8 MaxNatPredictionAuxiliaryStunURLs = 2 diff --git a/internal/server/protocol/contract_test.go b/internal/server/protocol/contract_test.go index b72438f..daf834d 100644 --- a/internal/server/protocol/contract_test.go +++ b/internal/server/protocol/contract_test.go @@ -75,7 +75,7 @@ const ( func authenticatedHost() []member { return []member{ {"type", `"authenticated"`}, - {"protocol", `"screener-v21"`}, + {"protocol", `"screener-v22"`}, {"peerId", `"host_12345678"`}, {"roomExpiresAt", `"2026-08-18T18:00:00.000Z"`}, {"maxViewers", `8`}, @@ -176,12 +176,6 @@ func TestAuthenticatedRoomLifetimeAndRevision(t *testing.T) { object(drop(authenticatedHost(), "routeAssignment")), false) assertServerMessage(t, "absent media mode", object(drop(authenticatedHost(), "mediaMode")), false) - assertServerMessage(t, "sfu standby url", - object(with(authenticatedHost(), - member{"sfuStandbyUrl", `"wss://sfu.example.test"`})), true) - assertServerMessage(t, "sfu standby url over https", - object(with(authenticatedHost(), - member{"sfuStandbyUrl", `"https://sfu.example.test"`})), false) assertServerMessage(t, "quality settings past the bitrate ceiling", object(with(authenticatedHost(), member{"qualitySettings", `{"resolution":"1080p","maxFramerate":60,"maxBitrate":20000000,` + @@ -479,11 +473,9 @@ func TestForwardedEvidenceAndSfuConfig(t *testing.T) { `"presentationEpoch":0},"sequence":0,"windowMs":2000,"metrics":`+metrics+`}`), false) assertServerMessage(t, "sfu config", - []byte(`{"type":"sfu-config","revision":9,"url":"wss://sfu.example.test",`+ - `"token":"header.payload.signature"}`), true) - assertServerMessage(t, "sfu config with an oversized token", - []byte(`{"type":"sfu-config","revision":9,"url":"wss://sfu.example.test","token":"`+ - repeat("x", MaxSfuTokenLength+1)+`"}`), false) + []byte(`{"type":"sfu-config","revision":9,"publicationGeneration":"publication_12345678","connectionId":"connection_12345678"}`), true) + assertServerMessage(t, "sfu config missing connection fence", + []byte(`{"type":"sfu-config","revision":9,"publicationGeneration":"publication_12345678"}`), false) } // zod's z.number().int() accepts an integral JSON number in any spelling. @@ -563,7 +555,6 @@ func TestEncodeServerMessageEmitsEmptyArrays(t *testing.T) { // The authenticated key order follows the object literal in signaling.ts. func TestAuthenticatedKeyOrder(t *testing.T) { - standby := "wss://sfu.example.test" paused := true encoded, err := EncodeServerMessage(AuthenticatedViewerMessage{ Type: "authenticated", Protocol: SignalingProtocol, PeerID: "viewer_12345678", @@ -571,7 +562,7 @@ func TestAuthenticatedKeyOrder(t *testing.T) { IceConfig: IceConfig{}, RoutePolicy: DefaultRoutePolicy, CodeEntryPolicy: CodeEntryOpen, ViewerAuthorizationGeneration: "viewer_generation_12345678", MediaMode: "peer-assisted", RouteRevision: 0, - QualitySettings: DefaultQualitySettings, SfuStandbyURL: &standby, Role: RoleViewer, + QualitySettings: DefaultQualitySettings, Role: RoleViewer, }) if err != nil { t.Fatal(err) @@ -597,7 +588,7 @@ func TestAuthenticatedKeyOrder(t *testing.T) { "endpointMediaCopyCapacity", "hostOnline", "hostPaused", "connectionId", "iceConfig", "routePolicy", "codeEntryPolicy", "viewerAuthorizationGeneration", "mediaMode", "shareGeneration", "routeRevision", "routeAssignment", - "qualitySettings", "sfuStandbyUrl", "role", + "qualitySettings", "role", } if strings.Join(order, ",") != strings.Join(want, ",") { t.Fatalf("key order\n got: %v\nwant: %v", order, want) @@ -631,7 +622,7 @@ func TestOptionalAndNullableAreExact(t *testing.T) { // A host authenticate without routePolicy takes DEFAULT_ROUTE_POLICY. func TestAuthenticateAppliesTheRoutePolicyDefault(t *testing.T) { message, err := DecodeClientMessage([]byte( - `{"type":"authenticate","protocol":"screener-v21","roomId":"1234","role":"host",` + + `{"type":"authenticate","protocol":"screener-v22","roomId":"1234","role":"host",` + `"token":"` + repeat("a", 43) + `","clientId":"client_12345678"}`)) if err != nil { t.Fatal(err) diff --git a/internal/server/protocol/fixture_test.go b/internal/server/protocol/fixture_test.go index caade3d..f02d054 100644 --- a/internal/server/protocol/fixture_test.go +++ b/internal/server/protocol/fixture_test.go @@ -48,7 +48,6 @@ func TestFixtureConstants(t *testing.T) { "maxSignalBytes": float64(MaxSignalBytes), "roomCodeLength": float64(RoomCodeLength), "maxMediaRouteRevision": float64(MaxMediaRouteRevision), - "maxSfuTokenLength": float64(MaxSfuTokenLength), "maxIceServerUrls": float64(MaxIceServerURLs), "maxNatPredictionAuxiliaryStunUrls": float64(MaxNatPredictionAuxiliaryStunURLs), "maxViewerQualityEvidenceBytes": float64(MaxViewerQualityEvidenceBytes), diff --git a/internal/server/protocol/scalars.go b/internal/server/protocol/scalars.go index 0f16624..18d9740 100644 --- a/internal/server/protocol/scalars.go +++ b/internal/server/protocol/scalars.go @@ -77,19 +77,6 @@ func ValidViewerPassword(value string) bool { viewerPasswordPattern.MatchString(value) } -// ValidLiveKitWebSocketURL mirrors liveKitWebSocketUrlSchema. -func ValidLiveKitWebSocketURL(value string) bool { - if UTF16Length(value) > 2048 { - return false - } - parsed, err := url.Parse(value) - if err != nil || parsed.Host == "" { - return false - } - scheme := strings.ToLower(parsed.Scheme) - return scheme == "ws" || scheme == "wss" -} - // isoDateTimePattern mirrors z.string().datetime(): an RFC 3339 UTC instant // with a "Z" designator and no offset. var isoDateTimePattern = regexp.MustCompile( diff --git a/internal/server/protocol/scalars_test.go b/internal/server/protocol/scalars_test.go index 2fb3346..f384f90 100644 --- a/internal/server/protocol/scalars_test.go +++ b/internal/server/protocol/scalars_test.go @@ -113,10 +113,6 @@ func TestScalarValidators(t *testing.T) { {"viewer password with newline", ValidViewerPassword, "line\nbreak", false}, {"viewer password past ceiling", ValidViewerPassword, repeat("x", MaxViewerPasswordLength+1), false}, - {"livekit url", ValidLiveKitWebSocketURL, "wss://sfu.example.test", true}, - {"livekit url plain ws", ValidLiveKitWebSocketURL, "ws://127.0.0.1:7880", true}, - {"livekit url https", ValidLiveKitWebSocketURL, "https://sfu.example.test", false}, - {"livekit url relative", ValidLiveKitWebSocketURL, "/signal", false}, } { t.Run(sample.name, func(t *testing.T) { if got := sample.valid(sample.value); got != sample.want { diff --git a/internal/server/protocol/server.go b/internal/server/protocol/server.go index c54c1dd..c0eea1f 100644 --- a/internal/server/protocol/server.go +++ b/internal/server/protocol/server.go @@ -34,7 +34,6 @@ type AuthenticatedHostMessage struct { RouteRevision Int `json:"routeRevision"` RouteAssignment ParticipantRouteAssignment `json:"routeAssignment"` QualitySettings QualitySettings `json:"qualitySettings"` - SfuStandbyURL *string `json:"sfuStandbyUrl,omitempty"` Role Role `json:"role"` ViewerPasswordEnabled bool `json:"viewerPasswordEnabled"` } @@ -60,7 +59,6 @@ type AuthenticatedViewerMessage struct { RouteRevision Int `json:"routeRevision"` RouteAssignment ParticipantRouteAssignment `json:"routeAssignment"` QualitySettings QualitySettings `json:"qualitySettings"` - SfuStandbyURL *string `json:"sfuStandbyUrl,omitempty"` Role Role `json:"role"` } @@ -117,12 +115,12 @@ type RouteDiagnosticSnapshotMessage struct { Snapshot RouteDiagnosticSnapshot `json:"snapshot"` } -// SfuConfigMessage carries a LiveKit URL and a short-lived token. +// SfuConfigMessage authorizes one embedded media connection. type SfuConfigMessage struct { - Type string `json:"type"` - Revision Int `json:"revision"` - URL string `json:"url"` - Token string `json:"token"` + Type string `json:"type"` + Revision Int `json:"revision"` + PublicationGeneration string `json:"publicationGeneration"` + ConnectionID string `json:"connectionId"` } // QualitySettingsMessage broadcasts the host's quality settings. @@ -136,7 +134,6 @@ type RoutePolicyMessage struct { Type string `json:"type"` ShareGeneration string `json:"shareGeneration"` RoutePolicy RoutePolicy `json:"routePolicy"` - SfuStandbyURL *string `json:"sfuStandbyUrl,omitempty"` } // PauseSharingSourceMessage asks the host to pause its capture source. @@ -255,6 +252,8 @@ func DecodeServerMessage(data []byte) (ServerMessage, error) { return decodeRouteDiagnosticSnapshotMessage(data) case "sfu-config": return decodeSfuConfig(data) + case "sfu-signal": + return decodeSfuSignal(data) case "quality-settings": return decodeQualitySettingsMessage(data) case "route-policy": @@ -298,7 +297,7 @@ func validateAuthenticatedBase( protocol string, peerID string, roomExpiresAt *string, maxViewers, endpointCapacity Int, connectionID *string, codeEntryPolicy, viewerAuthorizationGeneration, mediaMode string, - shareGeneration *string, routeRevision Int, sfuStandbyURL *string, + shareGeneration *string, routeRevision Int, ) error { if err := present.require("type", "protocol", "peerId", "maxViewers", "endpointMediaCopyCapacity", "hostOnline", "iceConfig", "codeEntryPolicy", @@ -310,7 +309,7 @@ func validateAuthenticatedBase( "roomExpiresAt", "connectionId", "shareGeneration"); err != nil { return err } - if err := present.optional("hostPaused", "routePolicy", "sfuStandbyUrl"); err != nil { + if err := present.optional("hostPaused", "routePolicy"); err != nil { return err } if protocol != SignalingProtocol { @@ -346,9 +345,6 @@ func validateAuthenticatedBase( if !validRevision(routeRevision) { return errors.New("routeRevision is out of range") } - if sfuStandbyURL != nil && !ValidLiveKitWebSocketURL(*sfuStandbyURL) { - return errors.New("sfuStandbyUrl is not a LiveKit WebSocket URL") - } return nil } @@ -371,7 +367,7 @@ func decodeAuthenticated(data []byte) (ServerMessage, error) { message.RoomExpiresAt, message.MaxViewers, message.EndpointMediaCopyCapacity, message.ConnectionID, message.CodeEntryPolicy, message.ViewerAuthorizationGeneration, message.MediaMode, - message.ShareGeneration, message.RouteRevision, message.SfuStandbyURL); err != nil { + message.ShareGeneration, message.RouteRevision); err != nil { return nil, err } if !present.has("routePolicy") { @@ -388,7 +384,7 @@ func decodeAuthenticated(data []byte) (ServerMessage, error) { message.RoomExpiresAt, message.MaxViewers, message.EndpointMediaCopyCapacity, message.ConnectionID, message.CodeEntryPolicy, message.ViewerAuthorizationGeneration, message.MediaMode, - message.ShareGeneration, message.RouteRevision, message.SfuStandbyURL); err != nil { + message.ShareGeneration, message.RouteRevision); err != nil { return nil, err } if !present.has("routePolicy") { @@ -521,17 +517,14 @@ func decodeSfuConfig(data []byte) (ServerMessage, error) { if err != nil { return nil, err } - if err := present.require("type", "revision", "url", "token"); err != nil { + if err := present.require("type", "revision", "publicationGeneration", "connectionId"); err != nil { return nil, err } if !validRevision(message.Revision) { return nil, errors.New("revision is out of range") } - if !ValidLiveKitWebSocketURL(message.URL) { - return nil, errors.New("url is not a LiveKit WebSocket URL") - } - if length := UTF16Length(message.Token); length < 1 || length > MaxSfuTokenLength { - return nil, errors.New("token length is out of range") + if !ValidOpaqueID(message.PublicationGeneration) || !ValidOpaqueID(message.ConnectionID) { + return nil, errors.New("invalid SFU connection fence") } return message, nil } @@ -557,15 +550,9 @@ func decodeRoutePolicyMessage(data []byte) (ServerMessage, error) { if err := present.require("type", "shareGeneration", "routePolicy"); err != nil { return nil, err } - if err := present.optional("sfuStandbyUrl"); err != nil { - return nil, err - } if !ValidOpaqueID(message.ShareGeneration) { return nil, errors.New("shareGeneration is not an opaque id") } - if message.SfuStandbyURL != nil && !ValidLiveKitWebSocketURL(*message.SfuStandbyURL) { - return nil, errors.New("sfuStandbyUrl is not a LiveKit WebSocket URL") - } return message, nil } diff --git a/internal/server/protocol/sfu.go b/internal/server/protocol/sfu.go new file mode 100644 index 0000000..5e40004 --- /dev/null +++ b/internal/server/protocol/sfu.go @@ -0,0 +1,115 @@ +package protocol + +import "errors" + +// SfuMedia describes the publisher's real encodings, ordered low to high. +type SfuMedia struct { + Codec string `json:"codec"` + Layers []SfuLayer `json:"layers"` + Audio bool `json:"audio"` + AudioBitrate Int `json:"audioBitrate"` +} + +type SfuLayer struct { + RID string `json:"rid"` + Width Int `json:"width"` + Height Int `json:"height"` + Bitrate Int `json:"bitrate"` +} + +func (value *SfuMedia) UnmarshalJSON(data []byte) error { + type raw SfuMedia + present, err := decodeObject(data, (*raw)(value)) + if err != nil { + return err + } + if err := present.require("codec", "layers", "audio", "audioBitrate"); err != nil { + return err + } + if !enumOf(value.Codec, "h264", "vp8") || len(value.Layers) < 1 || len(value.Layers) > 3 || + !inRangeInt(value.AudioBitrate, 0, 510_000) || value.Audio && value.AudioBitrate == 0 { + return errors.New("invalid SFU media") + } + for index, layer := range value.Layers { + if len(layer.RID) > 16 || layer.RID == "" && len(value.Layers) != 1 || + !inRangeInt(layer.Width, 1, 8192) || !inRangeInt(layer.Height, 1, 8192) || !inRangeInt(layer.Bitrate, 1, 100_000_000) { + return errors.New("invalid SFU layer") + } + for _, previous := range value.Layers[:index] { + if previous.RID == layer.RID { + return errors.New("duplicate SFU RID") + } + } + } + return nil +} + +func (value *SfuLayer) UnmarshalJSON(data []byte) error { + type raw SfuLayer + present, err := decodeObject(data, (*raw)(value)) + if err != nil { + return err + } + return present.require("rid", "width", "height", "bitrate") +} + +// SfuSignalMessage travels only on the room's authenticated WebSocket. +type SfuSignalMessage struct { + Type string `json:"type"` + Revision Int `json:"revision"` + PublicationGeneration string `json:"publicationGeneration"` + ConnectionID string `json:"connectionId"` + Kind string `json:"kind"` + Description *SessionDescription `json:"description,omitempty"` + Candidate *IceCandidate `json:"candidate,omitempty"` + Media *SfuMedia `json:"media,omitempty"` + ActiveCount *Int `json:"activeCount,omitempty"` +} + +func (SfuSignalMessage) isClientMessage() {} +func (SfuSignalMessage) isServerMessage() {} + +func decodeSfuSignal(data []byte) (SfuSignalMessage, error) { + var message SfuSignalMessage + present, err := decodeObject(data, &message) + if err != nil { + return message, err + } + if err := present.require("type", "revision", "publicationGeneration", "connectionId", "kind"); err != nil { + return message, err + } + if err := present.optional("description", "candidate", "media", "activeCount"); err != nil { + return message, err + } + if message.Kind != "layers" && message.ActiveCount != nil { + return message, errors.New("unexpected SFU layer demand") + } + if !validRevision(message.Revision) || !ValidOpaqueID(message.PublicationGeneration) || !ValidOpaqueID(message.ConnectionID) { + return message, errors.New("invalid SFU signal fence") + } + switch message.Kind { + case "layers": + if message.ActiveCount == nil || !inRangeInt(*message.ActiveCount, 0, 3) || message.Description != nil || message.Candidate != nil || message.Media != nil { + return message, errors.New("invalid SFU layer demand") + } + case "media": + if message.Media == nil || message.Description != nil || message.Candidate != nil { + return message, errors.New("invalid SFU media signal") + } + case "subscribe": + if message.Description != nil || message.Candidate != nil || message.Media != nil { + return message, errors.New("invalid SFU subscription signal") + } + case "description": + if message.Description == nil || message.Candidate != nil || message.Media != nil && message.Description.Type != "offer" { + return message, errors.New("invalid SFU description signal") + } + case "candidate": + if message.Description != nil || message.Media != nil { + return message, errors.New("invalid SFU ICE signal") + } + default: + return message, errors.New("unknown SFU signal kind") + } + return message, nil +} diff --git a/internal/server/route/api.go b/internal/server/route/api.go index ea887bd..bd1e755 100644 --- a/internal/server/route/api.go +++ b/internal/server/route/api.go @@ -20,6 +20,7 @@ type Resource struct { ShareGeneration string PublicationGeneration string ViewerPeerID string + ConnectionID string // Released is flipped by the router once the allocation is freed. Released bool } diff --git a/internal/server/route/debug.go b/internal/server/route/debug.go index 2d74eee..621f2d2 100644 --- a/internal/server/route/debug.go +++ b/internal/server/route/debug.go @@ -1,6 +1,7 @@ package route import ( + "context" "log/slog" "os" "strconv" @@ -9,9 +10,10 @@ import ( "github.com/TNTcraftHIM/Screener/internal/server/protocol" ) -// debugEnabled is the port of `debuglog("screener-route")`: read once, -// enabled by SCREENER_DEBUG=route (comma-separated like NODE_DEBUG). -var debugEnabled = routeDebugEnabled(os.Getenv("SCREENER_DEBUG")) +func debugEnabled() bool { + return routeDebugEnabled(os.Getenv("SCREENER_DEBUG")) || + slog.Default().Enabled(context.Background(), slog.LevelDebug) +} func routeDebugEnabled(value string) bool { for _, section := range strings.Split(value, ",") { @@ -28,13 +30,11 @@ type debugCandidate struct { Transition TransitionKind } -// debug ports 4911: one sanitised event with the room's debug ID, the -// current revision and fact version, then the caller's key/value details. +// debug emits one sanitized event with a diagnostic room ID and current facts. // Never pass raw peer IDs, session IDs or connection IDs; use debugPeer and -// debugTuple. It is a no-op without the env flag or a DebugRoomID; the -// level is Info because the env flag is the gate, as NODE_DEBUG was. +// debugTuple. The environment or diagnostic logger enables these Info events. func (c *Controller) debug(event string, details ...any) { - if !debugEnabled || c.debugRoomID == "" { + if !debugEnabled() || c.debugRoomID == "" { return } args := make([]any, 0, 8+len(details)) diff --git a/internal/server/route/debug_test.go b/internal/server/route/debug_test.go new file mode 100644 index 0000000..3344c56 --- /dev/null +++ b/internal/server/route/debug_test.go @@ -0,0 +1,24 @@ +package route + +import ( + "io" + "log/slog" + "testing" +) + +func TestDebugUsesCurrentEnvironmentAndLogger(t *testing.T) { + previous := slog.Default() + t.Cleanup(func() { slog.SetDefault(previous) }) + slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, nil))) + for value, want := range map[string]bool{"": false, "client": false, "route": true, "client, route": true} { + t.Setenv("SCREENER_DEBUG", value) + if got := debugEnabled(); got != want { + t.Fatalf("debugEnabled with %q = %v, want %v", value, got, want) + } + } + t.Setenv("SCREENER_DEBUG", "") + slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{Level: slog.LevelDebug}))) + if !debugEnabled() { + t.Fatal("the active diagnostic logger did not enable route events") + } +} diff --git a/internal/server/route/operation.go b/internal/server/route/operation.go index 5cfac59..954f5e1 100644 --- a/internal/server/route/operation.go +++ b/internal/server/route/operation.go @@ -214,7 +214,7 @@ func (c *Controller) Reconcile(nowMs int64) ReconcileResult { deferredParentPeerIDs: []string{}, qualitySource: qualitySource, } - if debugEnabled { + if debugEnabled() { routes := make([]debugCandidate, 0, len(candidates)) for _, candidate := range candidates { routes = append(routes, debugCandidate{Route: c.debugTuple(candidate.Tuple), Transition: candidate.EndpointTransition.Kind}) @@ -318,7 +318,7 @@ func (c *Controller) BeginCurrentCandidate(input BeginInput) BeginResult { mediaReady: false, connectionAttempt: c.startCandidateOpportunity(op, *plan), } - if debugEnabled { + if debugEnabled() { details := []any{ "child", c.debugPeer(op.childPeerID), "candidate", c.debugTuple(tuple), @@ -689,7 +689,7 @@ func (c *Controller) CandidateFailed(guard CandidateGuard, nowMs int64) SettleRe // OperationExpired applies head-start and deadline expiry at nowMs; // Accepted reports whether something expired. func (c *Controller) OperationExpired(nowMs int64) SettleResult { - if op := c.operation; op != nil && debugEnabled { + if op := c.operation; op != nil && debugEnabled() { current := op.current noProgress := current != nil && current.tuple.Kind == UpstreamPeer && diff --git a/internal/server/sfu/admission.go b/internal/server/sfu/admission.go index a59793e..02dbdd0 100644 --- a/internal/server/sfu/admission.go +++ b/internal/server/sfu/admission.go @@ -1,16 +1,8 @@ -// Package sfu ports the server's LiveKit-facing modules: the resource -// admission ledger (src/server/sfu-resource-admission.ts), the RoomService -// control plane (src/server/sfu-room-control.ts) and the participant token -// issuer (src/server/livekit-token.ts). -// -// LiveKit access is stdlib only (DECISIONS D1): one HS256 JWT signed with -// crypto/hmac and Twirp JSON over net/http, no SDK. +// Package sfu owns admitted physical publication and subscription resources. package sfu import ( - "errors" "regexp" - "strings" "github.com/TNTcraftHIM/Screener/internal/server/ordered" "github.com/TNTcraftHIM/Screener/internal/server/protocol" @@ -30,6 +22,7 @@ type ResourceFence struct { type SubscriptionFence struct { ResourceFence ViewerPeerID string + ConnectionID string } // Usage is SfuResourceUsage. @@ -72,8 +65,7 @@ type publicationEntry struct { // insertion order, both of which the router turns into drain tasks. type roomPublications = ordered.Map[string, *publicationEntry] -// Admission is SfuResourceAdmission: the ingress/egress ledger for the LiveKit -// fallback. It is not safe for concurrent use; the signaling server calls it +// Admission is SfuResourceAdmission: the ingress/egress ledger for the embedded SFU. It is not safe for concurrent use; the signaling server calls it // under its global mutex, exactly as Node's single thread did. type Admission struct { // Ordering site: beginDrainAll walks the rooms in insertion order. @@ -139,24 +131,15 @@ func (admission *Admission) ReserveSubscription(fence SubscriptionFence) bool { if publication == nil || publication.state == stateDraining { return false } - if existing, ok := publication.subscriptions[fence.ViewerPeerID]; ok { - if existing.fence != fence { - return false - } - if existing.state == stateReserved || existing.state == stateCommitted { - return true - } - if publication.state != stateCommitted { - return false - } - existing.state = stateReserved - return true + key := fence.ViewerPeerID + "\x00" + fence.ConnectionID + if existing, ok := publication.subscriptions[key]; ok { + return existing.fence == fence && existing.state != stateDraining } if admission.egressInUse >= admission.egressCapacity { return false } - publication.subscriptions[fence.ViewerPeerID] = &subscriptionEntry{ + publication.subscriptions[key] = &subscriptionEntry{ fence: fence, state: stateReserved, } @@ -217,7 +200,7 @@ func (admission *Admission) CommitSubscription(fence SubscriptionFence) bool { if publication == nil || publication.state != stateCommitted { return false } - subscription, ok := publication.subscriptions[fence.ViewerPeerID] + subscription, ok := publication.subscriptions[fence.ViewerPeerID+"\x00"+fence.ConnectionID] if !ok || subscription.fence != fence { return false } @@ -238,7 +221,7 @@ func (admission *Admission) BeginSubscriptionDrain(fence SubscriptionFence) bool if publication == nil { return false } - subscription, ok := publication.subscriptions[fence.ViewerPeerID] + subscription, ok := publication.subscriptions[fence.ViewerPeerID+"\x00"+fence.ConnectionID] if !ok || subscription.fence != fence { return false } @@ -257,6 +240,36 @@ func (admission *Admission) BeginDrain(fence ResourceFence) bool { return true } +func (admission *Admission) HasPublication(fence ResourceFence) bool { + publication := admission.publication(fence) + return publication != nil && publication.state != stateDraining +} + +func (admission *Admission) HasSubscription(fence SubscriptionFence) bool { + publication := admission.publication(fence.ResourceFence) + if publication == nil || publication.state == stateDraining { + return false + } + entry := publication.subscriptions[fence.ViewerPeerID+"\x00"+fence.ConnectionID] + return entry != nil && entry.fence == fence && entry.state != stateDraining +} + +// In-process closure revokes this exact connection; no reusable token remains. +func (admission *Admission) CompleteSubscriptionDrain(fence SubscriptionFence) bool { + publication := admission.publication(fence.ResourceFence) + if publication == nil { + return false + } + key := fence.ViewerPeerID + "\x00" + fence.ConnectionID + entry := publication.subscriptions[key] + if entry == nil || entry.fence != fence || entry.state != stateDraining { + return false + } + delete(publication.subscriptions, key) + admission.subtractEgress(1) + return true +} + // CompleteDrain ports completeDrain. func (admission *Admission) CompleteDrain(fence ResourceFence) bool { assertFence(fence) @@ -371,54 +384,10 @@ func assertPositiveSafeInteger(value int, name string) { } } -const managedRoomPrefix = "screener-v1." - -// roomIDPattern is ROOM_ID_PATTERN, /^[1-9]\d{0,11}$/. Go's \d and $ are the -// ASCII digits and end of text, as they are in a non-unicode, non-multiline -// JavaScript regular expression. OPAQUE_ID_PATTERN is protocol.ValidOpaqueID. var roomIDPattern = regexp.MustCompile(`^[1-9][0-9]{0,11}$`) -var errInvalidRoomFence = errors.New("Managed LiveKit room fence is invalid") - -// ManagedRoomName ports managedSfuRoomName. An invalid fence panics where the -// TypeScript throws, because the router builds room-name keys from it in -// synchronous code that never caught the throw. The RoomService calls, whose -// TypeScript throw was caught by the router, use the error-returning form. -func ManagedRoomName(fence ResourceFence) string { - name, err := managedRoomName(fence) - if err != nil { - panic(err.Error()) - } - return name -} - -func managedRoomName(fence ResourceFence) (string, error) { - if !roomIDPattern.MatchString(fence.RoomID) || - !protocol.ValidOpaqueID(fence.ShareGeneration) || - !protocol.ValidOpaqueID(fence.PublicationGeneration) { - return "", errInvalidRoomFence - } - return managedRoomPrefix + fence.RoomID + "." + - fence.ShareGeneration + "." + fence.PublicationGeneration, nil -} - -// IsManagedRoomName ports isManagedSfuRoomName. -func IsManagedRoomName(roomName string) bool { - rest, ok := strings.CutPrefix(roomName, managedRoomPrefix) - if !ok { - return false - } - parts := strings.Split(rest, ".") - return len(parts) == 3 && - roomIDPattern.MatchString(parts[0]) && - protocol.ValidOpaqueID(parts[1]) && - protocol.ValidOpaqueID(parts[2]) -} - -// viewerIdentity ports managedSfuViewerIdentity. -func viewerIdentity(peerID string) (string, error) { - if !protocol.ValidOpaqueID(peerID) { - return "", errors.New("Managed LiveKit Viewer identity is invalid") - } - return "viewer:" + peerID, nil +func validResourceFence(fence ResourceFence) bool { + return roomIDPattern.MatchString(fence.RoomID) && + protocol.ValidOpaqueID(fence.ShareGeneration) && + protocol.ValidOpaqueID(fence.PublicationGeneration) } diff --git a/internal/server/sfu/admission_test.go b/internal/server/sfu/admission_test.go index 7395b15..1f63c4e 100644 --- a/internal/server/sfu/admission_test.go +++ b/internal/server/sfu/admission_test.go @@ -10,6 +10,32 @@ import ( "github.com/TNTcraftHIM/Screener/internal/server/protocol" ) +func TestSubscriptionDrainReleasesOnlyExactConnection(t *testing.T) { + admission := NewAdmission(AdmissionOptions{IngressCapacity: 1, EgressCapacity: 2}) + publication := ResourceFence{RoomID: "1234", ShareGeneration: "share_12345678", PublicationGeneration: "publication_12345678"} + old := SubscriptionFence{ResourceFence: publication, ViewerPeerID: "viewer_12345678", ConnectionID: "old_12345678"} + current := old + current.ConnectionID = "new_12345678" + if !admission.ReservePublication(publication) || !admission.ReserveSubscription(old) { + t.Fatal("initial admission failed") + } + if _, ok := admission.CommitPublication(publication); !ok { + t.Fatal("publication commit failed") + } + if !admission.ReserveSubscription(current) { + t.Fatal("candidate connection was not charged independently") + } + if admission.Usage().Egress != 2 { + t.Fatal("two physical handles must remain charged") + } + if !admission.BeginSubscriptionDrain(old) || !admission.CompleteSubscriptionDrain(old) { + t.Fatal("exact subscription drain failed") + } + if admission.CompleteSubscriptionDrain(old) || !admission.HasSubscription(current) || admission.Usage().Egress != 1 { + t.Fatal("stale retirement changed the candidate") + } +} + func publicationFence(roomID string, publicationGeneration string) ResourceFence { return ResourceFence{ RoomID: roomID, @@ -215,7 +241,7 @@ func TestAdmissionKeepsReleasedReservedSubscriptionCharged(t *testing.T) { expectUsage(t, admission, 0, 0) } -func TestAdmissionReactivatesExactDrainingSubscription(t *testing.T) { +func TestAdmissionDoesNotReviveAConnectionDuringPhysicalClose(t *testing.T) { admission := NewAdmission(AdmissionOptions{IngressCapacity: 1, EgressCapacity: 1}) active := publicationFence("1", "publication_active") viewer := subscriptionFence(active, "viewer_a") @@ -230,18 +256,19 @@ func TestAdmissionReactivatesExactDrainingSubscription(t *testing.T) { } expectUsage(t, admission, 1, 1) - if !admission.ReserveSubscription(viewer) || !admission.ReserveSubscription(viewer) { - t.Fatal("the exact draining subscription must reactivate") + if admission.ReserveSubscription(viewer) { + t.Fatal("a draining connection cannot be revived") } expectUsage(t, admission, 1, 1) - if !admission.BeginSubscriptionDrain(viewer) { - t.Fatal("the subscription must drain again") - } if admission.ReserveSubscription(subscriptionFence(active, "viewer_b")) { t.Fatal("the charge must still block another viewer") } + if !admission.CompleteSubscriptionDrain(viewer) { + t.Fatal("physical close must release the charge") + } + viewer.ConnectionID = "replacement_12345678" if !admission.ReserveSubscription(viewer) || !admission.CommitSubscription(viewer) { - t.Fatal("the exact subscription must reactivate and commit") + t.Fatal("a new connection must reserve after physical close") } expectUsage(t, admission, 1, 1) } @@ -349,42 +376,3 @@ func TestAdmissionAssertsNoUnderflow(t *testing.T) { admission.CompleteDrain(active) }) } - -func TestManagedRoomName(t *testing.T) { - fence := ResourceFence{ - RoomID: "42", - ShareGeneration: "share_generation_12345678", - PublicationGeneration: "publication_generation_12345678", - } - name := ManagedRoomName(fence) - const want = "screener-v1.42.share_generation_12345678.publication_generation_12345678" - if name != want { - t.Fatalf("ManagedRoomName = %q, want %q", name, want) - } - if !IsManagedRoomName(name) { - t.Fatal("the generated name must be managed") - } - - for _, roomName := range []string{ - "screener-v1.42.bad.segment.extra", - "another-application", - "screener-v1.42.share_generation_12345678", - "screener-v1.042.share_generation_12345678.publication_generation_12345678", - "screener-v1.1234567890123.share_generation_12345678.publication_generation_12345678", - "screener-v1.42.short.publication_generation_12345678", - } { - if IsManagedRoomName(roomName) { - t.Fatalf("%q must not be a managed room name", roomName) - } - } - - for _, invalid := range []ResourceFence{ - {RoomID: "0", ShareGeneration: "share_generation_12345678", PublicationGeneration: "publication_generation_12345678"}, - {RoomID: "42", ShareGeneration: "has.dot.12345678", PublicationGeneration: "publication_generation_12345678"}, - {RoomID: "42", ShareGeneration: "share_generation_12345678", PublicationGeneration: "short"}, - } { - expectPanic(t, "Managed LiveKit room fence is invalid", func() { - ManagedRoomName(invalid) - }) - } -} diff --git a/internal/server/sfu/media.go b/internal/server/sfu/media.go new file mode 100644 index 0000000..6089415 --- /dev/null +++ b/internal/server/sfu/media.go @@ -0,0 +1,762 @@ +package sfu + +import ( + "context" + "errors" + "slices" + "strings" + "sync" + + "github.com/TNTcraftHIM/Screener/internal/media/forwarding" + "github.com/pion/interceptor" + "github.com/pion/rtcp" + "github.com/pion/webrtc/v4" +) + +// PublicationMedia is metadata from the authorized publisher, not a quality ladder. +type PublicationMedia struct { + Codec string + RIDs []string + Formats []forwarding.LayerFormat + Audio bool + AudioBitrate uint32 +} + +type MediaEvent struct { + Fence ResourceFence + ViewerPeerID string // Empty for the publisher. + ConnectionID string + Candidate *webrtc.ICECandidateInit // Nil with StateUnknown ends ICE gathering. + State webrtc.PeerConnectionState + ActiveCount *int +} + +type MediaOptions struct { + // Settings can contain the application-owned UDP mux. Media never closes it. + Settings webrtc.SettingEngine + // Events run without media locks and hand exact handles back to room authority. + // Physical close must be scheduled asynchronously from these callbacks. + Events func(MediaEvent) +} + +// Runtime is the physical boundary injected into the room router and its tests. +type Runtime interface { + HasPublication(ResourceFence, string) bool + PreparePublication(ResourceFence, string, PublicationMedia) error + UpdatePublication(ResourceFence, string, PublicationMedia) error + PublicationDemand(ResourceFence, string) (int, error) + AcceptPublisherOffer(ResourceFence, string, webrtc.SessionDescription) (webrtc.SessionDescription, error) + AddPublisherICE(ResourceFence, string, webrtc.ICECandidateInit) error + PrepareSubscriber(context.Context, SubscriptionFence, string) (webrtc.SessionDescription, error) + ApplySubscriberAnswer(SubscriptionFence, string, webrtc.SessionDescription) error + AddSubscriberICE(SubscriptionFence, string, webrtc.ICECandidateInit) error + RestartSubscriber(SubscriptionFence, string) (webrtc.SessionDescription, error) + CloseSubscription(SubscriptionFence, string) error + ClosePublication(ResourceFence) error +} + +// Media owns physical resources already authorized by the room's Admission. +// It does not admit rooms, issue credentials, retry, or choose route candidates. +type Media struct { + options MediaOptions + mu sync.Mutex + publications map[ResourceFence]*mediaPublication + closed bool + closeOnce sync.Once + closeErr error +} + +type mediaPublication struct { + owner *Media + fence ResourceFence + connectionID string + metadata PublicationMedia + pc *webrtc.PeerConnection + source *forwarding.Source + audio *webrtc.TrackLocalStaticRTP + ready chan struct{} + readyOnce sync.Once + mu sync.Mutex + signaling sync.Mutex + layers []bool + audioBound bool + subscribers map[subscriptionKey]*mediaSubscription + closed bool + failure error + readers sync.WaitGroup + prepares sync.WaitGroup + closeOnce sync.Once + closeErr error + demandWake chan struct{} + demandStop chan struct{} + demandDone chan struct{} +} + +type subscriptionKey struct { + peerID string + connectionID string +} + +type mediaSubscription struct { + key subscriptionKey + transport *forwarding.Transport + signaling sync.Mutex + ctx context.Context + cancel context.CancelFunc + prepared chan struct{} + closeOnce sync.Once + closeErr error +} + +func NewMedia(options MediaOptions) *Media { + return &Media{options: options, publications: make(map[ResourceFence]*mediaPublication)} +} + +func (media *Media) HasPublication(fence ResourceFence, connectionID string) bool { + _, err := media.publication(fence, connectionID) + return err == nil +} + +func validPublicationMedia(metadata PublicationMedia) bool { + if (metadata.Codec != "h264" && metadata.Codec != "vp8") || + len(metadata.Formats) < 1 || len(metadata.Formats) > 3 || + len(metadata.RIDs) != len(metadata.Formats) || + (metadata.Audio && metadata.AudioBitrate == 0) { + return false + } + for index, format := range metadata.Formats { + if format.Width == 0 || format.Height == 0 || format.Bitrate == 0 || + len(metadata.RIDs[index]) > 16 || + (metadata.RIDs[index] == "" && len(metadata.RIDs) != 1) || + slices.Contains(metadata.RIDs[:index], metadata.RIDs[index]) { + return false + } + } + return true +} + +// PreparePublication is called only after ReservePublication succeeds. +// Repeated preparation never replaces a live publication. +func (media *Media) PreparePublication(fence ResourceFence, connectionID string, metadata PublicationMedia) error { + if !validResourceFence(fence) || connectionID == "" || !validPublicationMedia(metadata) { + return errors.New("SFU publication media is invalid") + } + media.mu.Lock() + defer media.mu.Unlock() + if media.closed || media.publications[fence] != nil { + return errors.New("SFU publication is unavailable") + } + metadata.Formats = slices.Clone(metadata.Formats) + metadata.RIDs = slices.Clone(metadata.RIDs) + publication := &mediaPublication{ + owner: media, fence: fence, connectionID: connectionID, metadata: metadata, + ready: make(chan struct{}), layers: make([]bool, len(metadata.Formats)), + subscribers: make(map[subscriptionKey]*mediaSubscription), + demandWake: make(chan struct{}, 1), demandStop: make(chan struct{}), demandDone: make(chan struct{}), + } + engine := &webrtc.MediaEngine{} + if err := engine.RegisterDefaultCodecs(); err != nil { + return err + } + registry := &interceptor.Registry{} + if err := webrtc.RegisterDefaultInterceptors(engine, registry); err != nil { + return err + } + pc, err := webrtc.NewAPI(webrtc.WithMediaEngine(engine), webrtc.WithInterceptorRegistry(registry), + webrtc.WithSettingEngine(media.options.Settings)).NewPeerConnection(webrtc.Configuration{}) + if err != nil { + return err + } + publication.pc = pc + { + publication.audio, err = webrtc.NewTrackLocalStaticRTP(webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeOpus, ClockRate: 48_000, Channels: 2, + }, "audio", fence.PublicationGeneration) + if err != nil { + _ = pc.Close() + return err + } + } + pc.OnICECandidate(func(candidate *webrtc.ICECandidate) { + var value *webrtc.ICECandidateInit + if candidate != nil { + encoded := candidate.ToJSON() + value = &encoded + } + publication.emit(MediaEvent{Candidate: value}) + }) + pc.OnConnectionStateChange(func(state webrtc.PeerConnectionState) { + if state == webrtc.PeerConnectionStateFailed { + publication.fail(errors.New("SFU publisher transport failed")) + return + } + publication.emit(MediaEvent{State: state}) + }) + pc.OnTrack(publication.consumeTrack) + media.publications[fence] = publication + go publication.runDemand() + return nil +} + +func (publication *mediaPublication) wakeDemand() { + select { + case publication.demandWake <- struct{}{}: + default: + } +} + +func (publication *mediaPublication) requiredActiveCount() int { + publication.mu.Lock() + if publication.closed || publication.failure != nil { + publication.mu.Unlock() + return 0 + } + count, bootstrap := len(publication.metadata.Formats), false + transports := make([]*forwarding.Transport, 0, len(publication.subscribers)) + for _, subscriber := range publication.subscribers { + if subscriber.ctx.Err() != nil { + continue + } + if subscriber.transport == nil { + bootstrap = true + } else { + transports = append(transports, subscriber.transport) + } + } + publication.mu.Unlock() + if bootstrap { + return count + } + active := 0 + for _, transport := range transports { + switch transport.PC.ConnectionState() { + case webrtc.PeerConnectionStateNew, webrtc.PeerConnectionStateConnecting: + return count + case webrtc.PeerConnectionStateConnected, webrtc.PeerConnectionStateDisconnected: + active = max(active, transport.RequiredActiveCount()) + } + } + return min(active, count) +} + +func (publication *mediaPublication) runDemand() { + defer close(publication.demandDone) + previous := -1 + for { + select { + case <-publication.demandStop: + return + case <-publication.demandWake: + active := publication.requiredActiveCount() + if active != previous { + previous = active + publication.emit(MediaEvent{ActiveCount: &active}) + } + } + } +} + +func (media *Media) PublicationDemand(fence ResourceFence, connectionID string) (int, error) { + publication, err := media.publication(fence, connectionID) + if err != nil { + return 0, err + } + return publication.requiredActiveCount(), nil +} + +func (media *Media) publication(fence ResourceFence, connectionID string) (*mediaPublication, error) { + media.mu.Lock() + defer media.mu.Unlock() + publication := media.publications[fence] + if media.closed || publication == nil || + (connectionID != "" && publication.connectionID != connectionID) { + return nil, errors.New("SFU publication media is unavailable") + } + return publication, nil +} + +func (publication *mediaPublication) emit(event MediaEvent) { + publication.mu.Lock() + ended := publication.closed || publication.failure != nil && event.State != webrtc.PeerConnectionStateFailed + publication.mu.Unlock() + if ended { + return + } + owner := publication.owner + owner.mu.Lock() + current := !owner.closed && owner.publications[publication.fence] == publication + owner.mu.Unlock() + if current && owner.options.Events != nil { + event.Fence = publication.fence + if event.ConnectionID == "" { + event.ConnectionID = publication.connectionID + } + owner.options.Events(event) + } +} + +func (publication *mediaPublication) emitSubscriber(subscriber *mediaSubscription, event MediaEvent) { + publication.mu.Lock() + current := !publication.closed && publication.failure == nil && + publication.subscribers[subscriber.key] == subscriber && subscriber.ctx.Err() == nil + publication.mu.Unlock() + if current { + event.ViewerPeerID = subscriber.key.peerID + event.ConnectionID = subscriber.key.connectionID + publication.emit(event) + } +} + +func (media *Media) AcceptPublisherOffer(fence ResourceFence, connectionID string, + offer webrtc.SessionDescription) (webrtc.SessionDescription, error) { + publication, err := media.publication(fence, connectionID) + if err != nil { + return webrtc.SessionDescription{}, err + } + publication.signaling.Lock() + defer publication.signaling.Unlock() + if offer.Type != webrtc.SDPTypeOffer { + return webrtc.SessionDescription{}, errors.New("SFU publisher requires an offer") + } + if err = publication.pc.SetRemoteDescription(offer); err != nil { + return webrtc.SessionDescription{}, err + } + videoCount, audioCount := 0, 0 + for _, transceiver := range publication.pc.GetTransceivers() { + if transceiver.Direction() != webrtc.RTPTransceiverDirectionRecvonly && + transceiver.Direction() != webrtc.RTPTransceiverDirectionSendrecv { + continue + } + name := webrtc.MimeTypeOpus + if transceiver.Kind() == webrtc.RTPCodecTypeVideo { + videoCount++ + name = "video/" + publication.metadata.Codec + } else { + audioCount++ + } + parameters := transceiver.Receiver().GetParameters() + selected := slices.DeleteFunc(slices.Clone(parameters.Codecs), func(codec webrtc.RTPCodecParameters) bool { + return !strings.EqualFold(codec.MimeType, name) + }) + if len(selected) == 0 { + return webrtc.SessionDescription{}, errors.New("SFU publisher codec does not match its declaration") + } + if transceiver.Kind() == webrtc.RTPCodecTypeAudio { + selected[0].SDPFmtpLine = "minptime=10;useinbandfec=1;stereo=1;usedtx=0" + } + if err = transceiver.SetCodecPreferences(selected[:1]); err != nil { + return webrtc.SessionDescription{}, err + } + } + if videoCount != 1 || audioCount > 1 || publication.metadata.Audio && audioCount != 1 { + return webrtc.SessionDescription{}, errors.New("SFU publisher track count does not match its declaration") + } + answer, err := publication.pc.CreateAnswer(nil) + if err != nil { + return webrtc.SessionDescription{}, err + } + if err = publication.pc.SetLocalDescription(answer); err != nil { + return webrtc.SessionDescription{}, err + } + return answer, nil +} + +func (media *Media) AddPublisherICE(fence ResourceFence, connectionID string, candidate webrtc.ICECandidateInit) error { + publication, err := media.publication(fence, connectionID) + if err != nil { + return err + } + return publication.pc.AddICECandidate(candidate) +} + +func (media *Media) UpdatePublication(fence ResourceFence, connectionID string, metadata PublicationMedia) error { + publication, err := media.publication(fence, connectionID) + if err != nil { + return err + } + publication.signaling.Lock() + defer publication.signaling.Unlock() + publication.mu.Lock() + defer publication.mu.Unlock() + if publication.closed || publication.failure != nil || !validPublicationMedia(metadata) || + metadata.Codec != publication.metadata.Codec || !slices.Equal(metadata.RIDs, publication.metadata.RIDs) { + return errors.New("SFU media update changed the publication identity") + } + if publication.source != nil { + if err := publication.source.UpdateFormats(metadata.Formats); err != nil { + return err + } + } + metadata.Formats, metadata.RIDs = slices.Clone(metadata.Formats), slices.Clone(metadata.RIDs) + publication.metadata = metadata + publication.wakeDemand() + bitrate := uint32(0) + if metadata.Audio { + bitrate = metadata.AudioBitrate + } + for _, subscriber := range publication.subscribers { + if subscriber.transport != nil { + subscriber.transport.SetAudioBitrate(bitrate) + } + } + return nil +} + +func (publication *mediaPublication) consumeTrack(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver) { + video := track.Kind() == webrtc.RTPCodecTypeVideo + publication.mu.Lock() + layer := slices.Index(publication.metadata.RIDs, track.RID()) + if publication.closed || publication.failure != nil { + publication.mu.Unlock() + return + } + if video && (layer < 0 || publication.layers[layer] || + !strings.EqualFold(track.Codec().MimeType, "video/"+publication.metadata.Codec)) || + !video && (publication.audioBound || + !strings.EqualFold(track.Codec().MimeType, webrtc.MimeTypeOpus) || track.Codec().ClockRate != 48_000) { + publication.mu.Unlock() + publication.fail(errors.New("unexpected SFU publisher track")) + return + } + if video { + if publication.source == nil { + var err error + publication.source, err = forwarding.NewSource(forwarding.SourceOptions{ + ID: publication.fence.PublicationGeneration, StreamID: publication.fence.PublicationGeneration, + Codec: track.Codec(), Formats: publication.metadata.Formats, + HeaderExtensions: receiver.GetParameters().HeaderExtensions, + OnRTCP: func(_ int, packets []rtcp.Packet) { + // Pion's ingress interceptors already own NACK and receiver reports. + for _, packet := range packets { + switch packet.(type) { + case *rtcp.PictureLossIndication, *rtcp.FullIntraRequest: + _ = publication.pc.WriteRTCP([]rtcp.Packet{packet}) + } + } + }, + }) + if err != nil { + publication.mu.Unlock() + publication.fail(err) + return + } + } + if err := publication.source.BindLayer(layer, uint32(track.SSRC()), receiver.GetParameters()); err != nil { + publication.mu.Unlock() + publication.fail(err) + return + } + publication.layers[layer] = true + } else { + publication.audioBound = true + } + source := publication.source + publication.readers.Add(2) + if source != nil { + publication.readyOnce.Do(func() { close(publication.ready) }) + } + publication.mu.Unlock() + go publication.readRTCP(track, receiver, source, layer) + defer publication.readers.Done() + for { + packet, _, err := track.ReadRTP() + if err != nil { + return + } + if video { + err = source.WriteRTP(layer, packet) + } else { + // Extensions are negotiated per PC; downstream interceptors own theirs. + packet.Extension, packet.ExtensionProfile, packet.Extensions = false, 0, nil + // A connecting/retired subscriber owns its write failure, not the publisher. + _ = publication.audio.WriteRTP(packet) + } + if err != nil { + publication.fail(err) + return + } + } +} + +func (publication *mediaPublication) readRTCP(track *webrtc.TrackRemote, receiver *webrtc.RTPReceiver, + source *forwarding.Source, layer int) { + defer publication.readers.Done() + for { + var packets []rtcp.Packet + var err error + if track.RID() == "" { + packets, _, err = receiver.ReadRTCP() + } else { + packets, _, err = receiver.ReadSimulcastRTCP(track.RID()) + } + if err != nil { + return + } + if track.Kind() == webrtc.RTPCodecTypeVideo { + for _, packet := range packets { + if report, ok := packet.(*rtcp.SenderReport); ok && report.SSRC == uint32(track.SSRC()) { + _ = source.SenderReport(layer, report) + } + } + } + } +} + +func (publication *mediaPublication) fail(err error) { + publication.mu.Lock() + if publication.closed || publication.failure != nil { + publication.mu.Unlock() + return + } + publication.failure = err + publication.readyOnce.Do(func() { close(publication.ready) }) + publication.mu.Unlock() + // The callback may retire this handle; do not wait for the current reader in it. + go func() { + _ = publication.pc.Close() + publication.readers.Wait() + publication.emit(MediaEvent{State: webrtc.PeerConnectionStateFailed}) + }() +} + +// PrepareSubscriber follows ReserveSubscription. Its context is the existing +// route operation deadline, not a second media timeout. +func (media *Media) PrepareSubscriber(ctx context.Context, fence SubscriptionFence, + connectionID string) (_ webrtc.SessionDescription, err error) { + publication, err := media.publication(fence.ResourceFence, "") + if err != nil || fence.ViewerPeerID == "" || connectionID == "" { + return webrtc.SessionDescription{}, errors.New("SFU subscription media is unavailable") + } + key := subscriptionKey{peerID: fence.ViewerPeerID, connectionID: connectionID} + // The route-operation context bounds preparation, not committed media lifetime. + lifetime, cancel := context.WithCancel(context.Background()) + subscriber := &mediaSubscription{key: key, ctx: lifetime, cancel: cancel, prepared: make(chan struct{})} + publication.mu.Lock() + if publication.closed || publication.failure != nil || publication.subscribers[key] != nil { + publication.mu.Unlock() + cancel() + return webrtc.SessionDescription{}, errors.New("SFU subscription media is unavailable") + } + publication.subscribers[key] = subscriber + publication.wakeDemand() + publication.prepares.Add(1) + publication.mu.Unlock() + defer func() { + close(subscriber.prepared) + if err != nil { + _ = publication.closeSubscriber(subscriber) + } + publication.prepares.Done() + }() + select { + case <-publication.ready: + case <-ctx.Done(): + return webrtc.SessionDescription{}, ctx.Err() + case <-lifetime.Done(): + return webrtc.SessionDescription{}, lifetime.Err() + } + publication.mu.Lock() + valid := !publication.closed && publication.failure == nil && publication.source != nil && + publication.subscribers[key] == subscriber && ctx.Err() == nil && lifetime.Err() == nil + source := publication.source + failure := publication.failure + metadata := publication.metadata + publication.mu.Unlock() + if !valid { + return webrtc.SessionDescription{}, errors.Join(errors.New("SFU publication media is unavailable"), failure) + } + var audio webrtc.TrackLocal + if publication.audio != nil { + audio = publication.audio + } + transport, err := forwarding.NewTransport(forwarding.TransportOptions{ + Source: source, Settings: media.options.Settings, ConnectionID: fence.ViewerPeerID + ":" + connectionID, + InitialBitrate: int(metadata.Formats[len(metadata.Formats)-1].Bitrate), Audio: audio, + OnDemandChanged: publication.wakeDemand, + }) + if err != nil { + return webrtc.SessionDescription{}, err + } + if metadata.Audio { + transport.SetAudioBitrate(metadata.AudioBitrate) + } + publication.mu.Lock() + if publication.closed || publication.failure != nil || publication.subscribers[key] != subscriber || ctx.Err() != nil || lifetime.Err() != nil { + publication.mu.Unlock() + _ = transport.Close() + return webrtc.SessionDescription{}, errors.New("SFU subscription media is unavailable") + } + subscriber.transport = transport + publication.mu.Unlock() + transport.PC.OnICECandidate(func(candidate *webrtc.ICECandidate) { + var value *webrtc.ICECandidateInit + if candidate != nil { + encoded := candidate.ToJSON() + value = &encoded + } + publication.emitSubscriber(subscriber, MediaEvent{Candidate: value}) + }) + transport.PC.OnConnectionStateChange(func(state webrtc.PeerConnectionState) { + if state == webrtc.PeerConnectionStateConnected { + if err := transport.SetConnected(); err != nil { + state = webrtc.PeerConnectionStateFailed + } + } + publication.emitSubscriber(subscriber, MediaEvent{State: state}) + publication.wakeDemand() + }) + offer, err := transport.PC.CreateOffer(nil) + if err == nil { + err = transport.PC.SetLocalDescription(offer) + } + if err != nil { + return webrtc.SessionDescription{}, err + } + return offer, nil +} + +func (media *Media) subscriber(fence SubscriptionFence, connectionID string) (*mediaSubscription, error) { + publication, err := media.publication(fence.ResourceFence, "") + if err != nil { + return nil, err + } + publication.mu.Lock() + defer publication.mu.Unlock() + subscriber := publication.subscribers[subscriptionKey{peerID: fence.ViewerPeerID, connectionID: connectionID}] + if publication.closed || publication.failure != nil || subscriber == nil || subscriber.transport == nil || subscriber.ctx.Err() != nil { + return nil, errors.New("SFU subscription media is unavailable") + } + return subscriber, nil +} + +func (media *Media) ApplySubscriberAnswer(fence SubscriptionFence, connectionID string, answer webrtc.SessionDescription) error { + subscriber, err := media.subscriber(fence, connectionID) + if err != nil { + return err + } + subscriber.signaling.Lock() + defer subscriber.signaling.Unlock() + if answer.Type != webrtc.SDPTypeAnswer { + return errors.New("SFU subscriber requires an answer") + } + return subscriber.transport.PC.SetRemoteDescription(answer) +} + +// RestartSubscriber keeps the committed transport and asks ICE to recover it. +func (media *Media) RestartSubscriber(fence SubscriptionFence, connectionID string) (webrtc.SessionDescription, error) { + subscriber, err := media.subscriber(fence, connectionID) + if err != nil { + return webrtc.SessionDescription{}, err + } + subscriber.signaling.Lock() + defer subscriber.signaling.Unlock() + offer, err := subscriber.transport.PC.CreateOffer(&webrtc.OfferOptions{ICERestart: true}) + if err == nil { + err = subscriber.transport.PC.SetLocalDescription(offer) + } + return offer, err +} + +func (media *Media) AddSubscriberICE(fence SubscriptionFence, connectionID string, candidate webrtc.ICECandidateInit) error { + subscriber, err := media.subscriber(fence, connectionID) + if err != nil { + return err + } + return subscriber.transport.PC.AddICECandidate(candidate) +} + +func (media *Media) CloseSubscription(fence SubscriptionFence, connectionID string) error { + publication, err := media.publication(fence.ResourceFence, "") + if err != nil { + return nil + } + key := subscriptionKey{peerID: fence.ViewerPeerID, connectionID: connectionID} + publication.mu.Lock() + subscriber := publication.subscribers[key] + publication.mu.Unlock() + if subscriber != nil { + return publication.closeSubscriber(subscriber) + } + return nil +} + +func (publication *mediaPublication) closeSubscriber(subscriber *mediaSubscription) error { + subscriber.closeOnce.Do(func() { + subscriber.cancel() + <-subscriber.prepared + if subscriber.transport != nil { + subscriber.closeErr = subscriber.transport.Close() + } + }) + publication.mu.Lock() + if publication.subscribers[subscriber.key] == subscriber { + delete(publication.subscribers, subscriber.key) + publication.wakeDemand() + } + publication.mu.Unlock() + return subscriber.closeErr +} + +func (media *Media) ClosePublication(fence ResourceFence) error { + media.mu.Lock() + publication := media.publications[fence] + media.mu.Unlock() + if publication == nil { + return nil + } + err := publication.close() + media.mu.Lock() + if media.publications[fence] == publication { + delete(media.publications, fence) + } + media.mu.Unlock() + return err +} + +func (publication *mediaPublication) close() error { + publication.closeOnce.Do(func() { + publication.mu.Lock() + publication.closed = true + close(publication.demandStop) + publication.readyOnce.Do(func() { close(publication.ready) }) + var transports []*forwarding.Transport + var subscribers []*mediaSubscription + for _, subscriber := range publication.subscribers { + subscriber.cancel() + subscribers = append(subscribers, subscriber) + if subscriber.transport != nil { + transports = append(transports, subscriber.transport) + } + } + source := publication.source + publication.mu.Unlock() + publication.closeErr = publication.pc.Close() + for _, transport := range transports { + _ = transport.PC.Close() + } + for _, subscriber := range subscribers { + publication.closeErr = errors.Join(publication.closeErr, publication.closeSubscriber(subscriber)) + } + if source != nil { + source.Close() + } + publication.prepares.Wait() + publication.readers.Wait() + <-publication.demandDone + }) + return publication.closeErr +} + +func (media *Media) Close() error { + media.closeOnce.Do(func() { + media.mu.Lock() + media.closed = true + publications := media.publications + media.publications = nil + media.mu.Unlock() + for _, publication := range publications { + media.closeErr = errors.Join(media.closeErr, publication.close()) + } + }) + return media.closeErr +} diff --git a/internal/server/sfu/media_test.go b/internal/server/sfu/media_test.go new file mode 100644 index 0000000..9e5dcd9 --- /dev/null +++ b/internal/server/sfu/media_test.go @@ -0,0 +1,361 @@ +package sfu + +import ( + "context" + "fmt" + "net" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/TNTcraftHIM/Screener/internal/media/forwarding" + livekitsfu "github.com/livekit/livekit-server/pkg/sfu" + "github.com/pion/interceptor" + "github.com/pion/rtp" + "github.com/pion/rtp/codecs" + "github.com/pion/sdp/v3" + "github.com/pion/webrtc/v4" + "github.com/pion/webrtc/v4/pkg/media" +) + +func TestMediaOnePublicationTwoSubscribersAndExactRetirement(t *testing.T) { + for _, codec := range []string{"vp8", "h264"} { + for _, layerCount := range []int{1, 2} { + t.Run(fmt.Sprintf("%s/layers=%d", codec, layerCount), func(t *testing.T) { + runMediaFanout(t, codec, layerCount) + }) + } + } +} + +func TestMediaDemandBootstrapsWaitingSubscriberAndStopsAfterExactClose(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) + defer cancel() + demand := make(chan int, 4) + service := NewMedia(MediaOptions{Events: func(event MediaEvent) { + if event.ActiveCount != nil { + demand <- *event.ActiveCount + } + }}) + defer service.Close() + fence := ResourceFence{RoomID: "1234", ShareGeneration: "share_12345678", PublicationGeneration: "publication_12345678"} + metadata := PublicationMedia{Codec: "vp8", RIDs: []string{"q", "h"}, Formats: []forwarding.LayerFormat{{Width: 640, Height: 360, Bitrate: 500_000}, {Width: 1280, Height: 720, Bitrate: 2_000_000}}} + if err := service.PreparePublication(fence, "publisher_12345678", metadata); err != nil { + t.Fatal(err) + } + subscription := SubscriptionFence{ResourceFence: fence, ViewerPeerID: "viewer_12345678", ConnectionID: "connection_12345678"} + prepared := make(chan error, 1) + go func() { + _, err := service.PrepareSubscriber(ctx, subscription, subscription.ConnectionID) + prepared <- err + }() + select { + case count := <-demand: + if count != 2 { + t.Fatalf("bootstrap demand = %d", count) + } + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + if err := service.CloseSubscription(subscription, subscription.ConnectionID); err != nil { + t.Fatal(err) + } + select { + case err := <-prepared: + if err == nil { + t.Fatal("retired subscriber acquired a later input") + } + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + select { + case count := <-demand: + if count != 0 { + t.Fatalf("no-subscriber demand = %d", count) + } + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } +} + +func runMediaFanout(t *testing.T, codec string, layerCount int) { + t.Helper() + ctx, cancel := context.WithTimeout(t.Context(), 8*time.Second) + defer cancel() + check := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + settings := webrtc.SettingEngine{} + settings.SetNetworkTypes([]webrtc.NetworkType{webrtc.NetworkTypeUDP4}) + settings.SetIncludeLoopbackCandidate(true) + settings.SetIPFilter(func(ip net.IP) bool { return ip.IsLoopback() }) + newPeer := func() *webrtc.PeerConnection { + engine := &webrtc.MediaEngine{} + check(engine.RegisterDefaultCodecs()) + registry := &interceptor.Registry{} + check(webrtc.RegisterDefaultInterceptors(engine, registry)) + pc, err := webrtc.NewAPI(webrtc.WithMediaEngine(engine), webrtc.WithInterceptorRegistry(registry), + webrtc.WithSettingEngine(settings)).NewPeerConnection(webrtc.Configuration{}) + check(err) + t.Cleanup(func() { _ = pc.Close() }) + return pc + } + waitGathering := func(pc *webrtc.PeerConnection, description webrtc.SessionDescription) webrtc.SessionDescription { + gather := webrtc.GatheringCompletePromise(pc) + check(pc.SetLocalDescription(description)) + select { + case <-gather: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + return *pc.LocalDescription() + } + service := NewMedia(MediaOptions{Settings: settings}) + defer service.Close() + fence := ResourceFence{RoomID: "1234", ShareGeneration: "share_123456", PublicationGeneration: "publication_123456"} + admission := NewAdmission(AdmissionOptions{IngressCapacity: 1, EgressCapacity: 2}) + if !admission.ReservePublication(fence) { + t.Fatal("publication admission failed") + } + metadata := PublicationMedia{Codec: codec, RIDs: []string{""}, + Formats: []forwarding.LayerFormat{{Width: 8, Height: 8, Bitrate: 1_000_000}}, + Audio: true, AudioBitrate: 64_000} + if layerCount == 2 { + metadata.RIDs = []string{"q", "f"} + metadata.Formats = []forwarding.LayerFormat{ + {Width: 8, Height: 8, Bitrate: 250_000}, + {Width: 8, Height: 8, Bitrate: 1_000_000}, + } + } + check(service.PreparePublication(fence, "publisher_123456", metadata)) + if service.PreparePublication(fence, "replacement_123456", metadata) == nil { + t.Fatal("duplicate prepare replaced a publication") + } + publication, err := service.publication(fence, "publisher_123456") + check(err) + waitingFence := SubscriptionFence{ResourceFence: fence, ViewerPeerID: "viewer_123456"} + if !admission.ReserveSubscription(waitingFence) { + t.Fatal("waiting subscription admission failed") + } + waiting := make(chan error, 1) + go func() { + _, err := service.PrepareSubscriber(ctx, waitingFence, "canceled_123456") + waiting <- err + }() + for { + publication.mu.Lock() + registered := len(publication.subscribers) == 1 + publication.mu.Unlock() + if registered { + break + } + select { + case <-time.After(time.Millisecond): + case <-ctx.Done(): + t.Fatal("subscription was not reserved before input readiness") + } + } + check(service.CloseSubscription(waitingFence, "canceled_123456")) + select { + case err = <-waiting: + if err == nil { + t.Fatal("canceled subscription acquired a later source") + } + case <-ctx.Done(): + t.Fatal("retired subscription kept waiting for input") + } + publisher := newPeer() + capability := webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8, ClockRate: 90_000} + videoData := livekitsfu.VP8KeyFrame8x8 + if codec == "h264" { + capability.MimeType = webrtc.MimeTypeH264 + capability.SDPFmtpLine = "level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f" + videoData = nil + for _, nal := range livekitsfu.H264KeyFrame2x2 { + videoData = append(videoData, 0, 0, 0, 1) + videoData = append(videoData, nal...) + } + } + var videos []*webrtc.TrackLocalStaticRTP + var packetizers []rtp.Packetizer + for _, rid := range metadata.RIDs { + video, err := webrtc.NewTrackLocalStaticRTP(capability, "video", "fixture", webrtc.WithRTPStreamID(rid)) + check(err) + videos = append(videos, video) + var payloader rtp.Payloader = &codecs.VP8Payloader{EnablePictureID: true} + if codec == "h264" { + payloader = &codecs.H264Payloader{} + } + packetizers = append(packetizers, rtp.NewPacketizer(1200, 96, uint32(len(videos)), + payloader, rtp.NewRandomSequencer(), 90_000)) + } + audio, err := webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{ + MimeType: webrtc.MimeTypeOpus, ClockRate: 48_000, Channels: 2, + }, "audio", "fixture") + check(err) + videoSender, err := publisher.AddTrack(videos[0]) + check(err) + for _, video := range videos[1:] { + check(videoSender.AddEncoding(video)) + } + _, err = publisher.AddTrack(audio) + check(err) + for _, sender := range publisher.GetSenders() { + go func() { + for { + if _, _, readErr := sender.ReadRTCP(); readErr != nil { + return + } + } + }() + } + offer, err := publisher.CreateOffer(nil) + check(err) + _, err = service.AcceptPublisherOffer(fence, "publisher_123456", waitGathering(publisher, offer)) + check(err) + select { + case <-webrtc.GatheringCompletePromise(publication.pc): + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + check(publisher.SetRemoteDescription(*publication.pc.LocalDescription())) + mid := publisher.GetTransceivers()[0].Mid() + headerExtensions := videoSender.GetParameters().HeaderExtensions + stopSamples := make(chan struct{}) + var samples sync.WaitGroup + samples.Add(1) + go func() { + defer samples.Done() + ticker := time.NewTicker(time.Second / 30) + defer ticker.Stop() + for { + select { + case <-ticker.C: + for index, video := range videos { + for _, packet := range packetizers[index].Packetize(videoData, 3000) { + if layerCount > 1 { + for _, extension := range headerExtensions { + switch extension.URI { + case sdp.SDESMidURI: + _ = packet.SetExtension(uint8(extension.ID), []byte(mid)) + case sdp.SDESRTPStreamIDURI: + _ = packet.SetExtension(uint8(extension.ID), []byte(metadata.RIDs[index])) + } + } + } + _ = video.WriteRTP(packet) + } + } + _ = audio.WriteSample(media.Sample{Data: []byte{0xf8, 0xff, 0xfe}, Duration: 20 * time.Millisecond}) + case <-stopSamples: + return + } + } + }() + defer func() { close(stopSamples); samples.Wait() }() + var videoPackets, audioPackets [2]atomic.Uint32 + fences := []SubscriptionFence{ + {ResourceFence: fence, ViewerPeerID: "viewer_123456"}, + {ResourceFence: fence, ViewerPeerID: "viewer_abcdef"}, + } + var transports [2]*forwarding.Transport + for index, subscription := range fences { + if !admission.ReserveSubscription(subscription) { + t.Fatal("subscription admission failed") + } + offer, prepareErr := service.PrepareSubscriber(ctx, subscription, "subscriber_123456") + check(prepareErr) + physical, getErr := service.subscriber(subscription, "subscriber_123456") + check(getErr) + transports[index] = physical.transport + select { + case <-webrtc.GatheringCompletePromise(physical.transport.PC): + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + offer = *physical.transport.PC.LocalDescription() + receiver := newPeer() + receiver.OnTrack(func(track *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { + for { + packet, _, readErr := track.ReadRTP() + if readErr != nil { + return + } + if len(packet.Payload) == 0 { + continue + } + if track.Kind() == webrtc.RTPCodecTypeVideo { + videoPackets[index].Add(1) + } else { + audioPackets[index].Add(1) + } + } + }) + check(receiver.SetRemoteDescription(offer)) + answer, answerErr := receiver.CreateAnswer(nil) + check(answerErr) + check(service.ApplySubscriberAnswer(subscription, "subscriber_123456", waitGathering(receiver, answer))) + } + for videoPackets[0].Load() < 3 || videoPackets[1].Load() < 3 || + audioPackets[0].Load() < 3 || audioPackets[1].Load() < 3 { + select { + case <-time.After(10 * time.Millisecond): + case <-ctx.Done(): + t.Fatalf("missing media: video=%d,%d audio=%d,%d", + videoPackets[0].Load(), videoPackets[1].Load(), audioPackets[0].Load(), audioPackets[1].Load()) + } + } + if len(publisher.GetSenders()) != 2 || len(publication.source.GetDownTracks()) != 2 { + t.Fatal("one video/audio publication did not share its source") + } + publication.mu.Lock() + bound := 0 + for _, active := range publication.layers { + if active { + bound++ + } + } + publication.mu.Unlock() + if bound != layerCount { + t.Fatalf("negotiated RTP layers = %d, want %d", bound, layerCount) + } + originalSource := publication.source + updated := metadata + updated.Formats = append([]forwarding.LayerFormat(nil), metadata.Formats...) + updated.Formats[len(updated.Formats)-1].Bitrate /= 2 + updated.Audio = false + check(service.UpdatePublication(fence, "publisher_123456", updated)) + if publication.source != originalSource || publication.pc.ConnectionState() != webrtc.PeerConnectionStateConnected || + len(publication.source.GetDownTracks()) != 2 { + t.Fatal("live metadata update replaced physical media") + } + updated.Codec = "unsupported" + if service.UpdatePublication(fence, "publisher_123456", updated) == nil { + t.Fatal("metadata update changed the publication codec") + } + check(service.UpdatePublication(fence, "publisher_123456", metadata)) + check(service.CloseSubscription(fences[0], "stale_connection")) + if transports[0].PC.ConnectionState() != webrtc.PeerConnectionStateConnected { + t.Fatal("stale subscription identity closed current media") + } + check(service.CloseSubscription(fences[0], "subscriber_123456")) + if transports[0].PC.ConnectionState() != webrtc.PeerConnectionStateClosed || + transports[1].PC.ConnectionState() != webrtc.PeerConnectionStateConnected || + len(publication.source.GetDownTracks()) != 1 { + t.Fatal("subscription retirement disturbed its sibling") + } + if service.AddSubscriberICE(fences[0], "subscriber_123456", webrtc.ICECandidateInit{}) == nil { + t.Fatal("retired media accepted signaling") + } + check(service.ClosePublication(fence)) + check(service.ClosePublication(fence)) + if publication.pc.ConnectionState() != webrtc.PeerConnectionStateClosed || + transports[1].PC.ConnectionState() != webrtc.PeerConnectionStateClosed || + !publication.source.IsClosed() { + t.Fatal("publication did not retire its complete physical resources") + } +} diff --git a/internal/server/sfu/roomcontrol.go b/internal/server/sfu/roomcontrol.go deleted file mode 100644 index 0c72ec9..0000000 --- a/internal/server/sfu/roomcontrol.go +++ /dev/null @@ -1,189 +0,0 @@ -package sfu - -// Ported from src/server/sfu-room-control.ts. - -import ( - "context" - "errors" -) - -// RoomControl is SfuRoomControl. Every method performs network I/O, so it must -// be called with the signaling server's mutex released. -type RoomControl interface { - Initialize(ctx context.Context) error - CreateRoom(ctx context.Context, fence ResourceFence) error - DeleteRoom(ctx context.Context, fence ResourceFence) error - DrainSubscription(ctx context.Context, fence SubscriptionFence) error - HostParticipantExists(ctx context.Context, fence ResourceFence) (bool, error) -} - -// LiveKitRoomControlOptions is LiveKitSfuRoomControlOptions minus the injected -// roomService: the tests drive a real Twirp client against httptest. -type LiveKitRoomControlOptions struct { - APIURL string - APIKey string - APISecret string - MaxViewersPerRoom int -} - -// LiveKitRoomControl is LiveKitSfuRoomControl. It holds no state of its own: -// the TypeScript per-room-name operation queue moved to the router, which -// reserves each managed room's turn while the signaling mutex is still held -// and so orders the mutating calls by the decision that made them. -type LiveKitRoomControl struct { - service roomService - maxParticipants int -} - -var _ RoomControl = (*LiveKitRoomControl)(nil) - -// NewLiveKitRoomControl mirrors the LiveKitSfuRoomControl constructor. -func NewLiveKitRoomControl(options LiveKitRoomControlOptions) (*LiveKitRoomControl, error) { - if options.APIURL == "" || options.APIKey == "" || - len(options.APISecret) < minAPISecretBytes { - return nil, errors.New("LiveKit RoomService configuration is invalid") - } - if err := validateViewerLimit(options.MaxViewersPerRoom); err != nil { - return nil, err - } - return &LiveKitRoomControl{ - service: newRoomService(options.APIURL, newSigner(options.APIKey, options.APISecret)), - maxParticipants: options.MaxViewersPerRoom + 1, - }, nil -} - -// Initialize ports initialize(): it deletes in the order listRooms returned. -func (control *LiveKitRoomControl) Initialize(ctx context.Context) error { - rooms, err := control.service.listRooms(ctx, nil) - if err != nil { - return err - } - for _, room := range rooms { - if !IsManagedRoomName(room.Name) { - return errors.New("Dedicated LiveKit instance contains a foreign room") - } - } - for _, room := range rooms { - if err := control.deleteRoomByName(ctx, room.Name); err != nil { - return err - } - } - remaining, err := control.service.listRooms(ctx, nil) - if err != nil { - return err - } - if len(remaining) != 0 { - return errors.New("Dedicated LiveKit instance is not empty after startup drain") - } - return nil -} - -// CreateRoom ports createRoom(). -func (control *LiveKitRoomControl) CreateRoom(ctx context.Context, fence ResourceFence) error { - roomName, err := managedRoomName(fence) - if err != nil { - return err - } - existing, err := control.service.listRooms(ctx, []string{roomName}) - if err != nil { - return err - } - if len(existing) != 0 { - return errors.New("Managed LiveKit room already exists") - } - if err := control.service.createRoom(ctx, roomName, control.maxParticipants); err != nil { - return err - } - rooms, err := control.service.listRooms(ctx, []string{roomName}) - if err != nil { - return err - } - if len(rooms) != 1 || rooms[0].Name != roomName { - return errors.New("Managed LiveKit room creation was not confirmed") - } - return nil -} - -// DeleteRoom ports deleteRoom(). -func (control *LiveKitRoomControl) DeleteRoom(ctx context.Context, fence ResourceFence) error { - roomName, err := managedRoomName(fence) - if err != nil { - return err - } - if err := control.deleteRoomByName(ctx, roomName); err != nil { - return err - } - rooms, err := control.service.listRooms(ctx, []string{roomName}) - if err != nil { - return err - } - if len(rooms) != 0 { - return errors.New("Managed LiveKit room deletion was not confirmed") - } - return nil -} - -// DrainSubscription ports drainSubscription(). -func (control *LiveKitRoomControl) DrainSubscription(ctx context.Context, fence SubscriptionFence) error { - roomName, err := managedRoomName(fence.ResourceFence) - if err != nil { - return err - } - identity, err := viewerIdentity(fence.ViewerPeerID) - if err != nil { - return err - } - if err := control.service.removeParticipant(ctx, roomName, identity); err != nil && - !isNotFound(err) { - return err - } - exists, err := control.participantExists(ctx, roomName, identity) - if err != nil { - return err - } - if exists { - return errors.New("Managed LiveKit participant removal was not confirmed") - } - return nil -} - -// HostParticipantExists ports hostParticipantExists(). -func (control *LiveKitRoomControl) HostParticipantExists( - ctx context.Context, - fence ResourceFence, -) (bool, error) { - roomName, err := managedRoomName(fence) - if err != nil { - return false, err - } - return control.participantExists(ctx, roomName, "host") -} - -// deleteRoomByName ports deleteRoomByName(): a missing room is not an error. -func (control *LiveKitRoomControl) deleteRoomByName(ctx context.Context, roomName string) error { - if err := control.service.deleteRoom(ctx, roomName); err != nil && !isNotFound(err) { - return err - } - return nil -} - -// participantExists ports participantExists(): an absent room reads as absent. -func (control *LiveKitRoomControl) participantExists( - ctx context.Context, - roomName string, - identity string, -) (bool, error) { - participants, err := control.service.listParticipants(ctx, roomName) - if err != nil { - if isNotFound(err) { - return false, nil - } - return false, err - } - for _, participant := range participants { - if participant.Identity == identity { - return true, nil - } - } - return false, nil -} diff --git a/internal/server/sfu/roomcontrol_test.go b/internal/server/sfu/roomcontrol_test.go deleted file mode 100644 index 422e618..0000000 --- a/internal/server/sfu/roomcontrol_test.go +++ /dev/null @@ -1,438 +0,0 @@ -package sfu - -// Ported from tests/sfu-room-control.test.ts. The TypeScript injected a -// FakeRoomService; here the real Twirp client runs against an httptest server -// that reproduces that fake's behaviour, so the wire contract recorded in -// evidence/livekit.md section 3 is covered by the same scenarios. - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "os" - "regexp" - "slices" - "strings" - "sync" - "testing" - - "github.com/TNTcraftHIM/Screener/internal/server/ordered" -) - -// fakeLiveKit is tests/sfu-room-control.test.ts's FakeRoomService behind the -// real Twirp endpoints. -type fakeLiveKit struct { - mu sync.Mutex - // Ordering site: listRooms answers in insertion order, which initialize - // turns into its delete order. - rooms ordered.Map[string, map[string]struct{}] - // operations records the mutating calls, as the TypeScript fake did; - // listCalls counts the reads it did not record. - operations []string - listCalls int - createOptions []int - retainDeletedRoom bool - retainRemovedParticipant bool -} - -func newFakeLiveKit(t *testing.T) (*fakeLiveKit, string) { - t.Helper() - fake := &fakeLiveKit{} - server := httptest.NewServer(http.HandlerFunc(fake.serve)) - t.Cleanup(server.Close) - return fake, server.URL -} - -func (fake *fakeLiveKit) serve(writer http.ResponseWriter, request *http.Request) { - var body struct { - Name string `json:"name"` - Names []string `json:"names"` - Room string `json:"room"` - Identity string `json:"identity"` - MaxParticipants int `json:"maxParticipants"` - } - _ = json.NewDecoder(request.Body).Decode(&body) - method := request.URL.Path[strings.LastIndexByte(request.URL.Path, '/')+1:] - - status, payload := http.StatusOK, "{}" - switch method { - case "ListRooms": - payload = fake.listRooms(body.Names) - case "CreateRoom": - status, payload = fake.createRoom(body.Name, body.MaxParticipants) - case "DeleteRoom": - status, payload = fake.deleteRoom(body.Room) - case "ListParticipants": - status, payload = fake.listParticipants(body.Room) - case "RemoveParticipant": - status, payload = fake.removeParticipant(body.Room, body.Identity) - default: - status, payload = http.StatusNotFound, `{"code":"bad_route","msg":"unknown method"}` - } - writer.Header().Set("Content-Type", "application/json") - writer.WriteHeader(status) - _, _ = writer.Write([]byte(payload)) -} - -const notFoundBody = `{"code":"not_found","msg":"missing"}` - -func (fake *fakeLiveKit) listRooms(names []string) string { - fake.mu.Lock() - defer fake.mu.Unlock() - fake.listCalls++ - selected := make([]string, 0, fake.rooms.Len()) - for _, name := range fake.rooms.Keys() { - if len(names) == 0 || slices.Contains(names, name) { - selected = append(selected, `{"name":`+quote(name)+`}`) - } - } - return `{"rooms":[` + strings.Join(selected, ",") + `]}` -} - -func (fake *fakeLiveKit) createRoom(name string, maxParticipants int) (int, string) { - fake.mu.Lock() - fake.operations = append(fake.operations, "create:"+name) - fake.createOptions = append(fake.createOptions, maxParticipants) - defer fake.mu.Unlock() - if fake.rooms.Has(name) { - return http.StatusConflict, `{"code":"already_exists","msg":"room already exists"}` - } - fake.rooms.Set(name, map[string]struct{}{}) - return http.StatusOK, `{"name":` + quote(name) + `}` -} - -func (fake *fakeLiveKit) deleteRoom(room string) (int, string) { - fake.mu.Lock() - defer fake.mu.Unlock() - fake.operations = append(fake.operations, "delete:"+room) - if !fake.rooms.Has(room) { - return http.StatusNotFound, notFoundBody - } - if !fake.retainDeletedRoom { - fake.rooms.Delete(room) - } - return http.StatusOK, "{}" -} - -func (fake *fakeLiveKit) listParticipants(room string) (int, string) { - fake.mu.Lock() - defer fake.mu.Unlock() - participants, ok := fake.rooms.Get(room) - if !ok { - return http.StatusNotFound, notFoundBody - } - records := make([]string, 0, len(participants)) - for identity := range participants { - records = append(records, `{"identity":`+quote(identity)+`}`) - } - return http.StatusOK, `{"participants":[` + strings.Join(records, ",") + `]}` -} - -func (fake *fakeLiveKit) removeParticipant(room string, identity string) (int, string) { - fake.mu.Lock() - defer fake.mu.Unlock() - fake.operations = append(fake.operations, "remove:"+room+":"+identity) - participants, ok := fake.rooms.Get(room) - if !ok { - return http.StatusNotFound, notFoundBody - } - if _, joined := participants[identity]; !joined { - return http.StatusNotFound, notFoundBody - } - if !fake.retainRemovedParticipant { - delete(participants, identity) - } - return http.StatusOK, "{}" -} - -// join is the TypeScript fake's join(): it fails when the room is absent. -func (fake *fakeLiveKit) join(room string, identity string) bool { - fake.mu.Lock() - defer fake.mu.Unlock() - participants, ok := fake.rooms.Get(room) - if !ok { - return false - } - participants[identity] = struct{}{} - return true -} - -func (fake *fakeLiveKit) seed(room string, identities ...string) { - participants := make(map[string]struct{}, len(identities)) - for _, identity := range identities { - participants[identity] = struct{}{} - } - fake.mu.Lock() - defer fake.mu.Unlock() - fake.rooms.Set(room, participants) -} - -func (fake *fakeLiveKit) snapshot() ([]string, []string) { - fake.mu.Lock() - defer fake.mu.Unlock() - return fake.rooms.Keys(), append([]string(nil), fake.operations...) -} - -func (fake *fakeLiveKit) operationLog() []string { - _, operations := fake.snapshot() - return operations -} - -func (fake *fakeLiveKit) participants(room string) []string { - fake.mu.Lock() - defer fake.mu.Unlock() - found, ok := fake.rooms.Get(room) - if !ok { - return nil - } - identities := make([]string, 0, len(found)) - for identity := range found { - identities = append(identities, identity) - } - slices.Sort(identities) - return identities -} - -func quote(value string) string { - encoded, err := json.Marshal(value) - if err != nil { - panic(err) - } - return string(encoded) -} - -func testControl(t *testing.T, apiURL string) *LiveKitRoomControl { - t.Helper() - control, err := NewLiveKitRoomControl(LiveKitRoomControlOptions{ - APIURL: apiURL, - APIKey: testAPIKey, - APISecret: testAPISecret, - MaxViewersPerRoom: 8, - }) - if err != nil { - t.Fatalf("NewLiveKitRoomControl: %v", err) - } - return control -} - -func testFence() ResourceFence { - return ResourceFence{ - RoomID: "42", - ShareGeneration: testShareGeneration, - PublicationGeneration: "publication_generation_12345678", - } -} - -func TestRoomControlDrainsManagedRoomsBeforeStartup(t *testing.T) { - fake, apiURL := newFakeLiveKit(t) - current := ManagedRoomName(testFence()) - fake.seed(current, "host") - - if err := testControl(t, apiURL).Initialize(context.Background()); err != nil { - t.Fatalf("Initialize: %v", err) - } - rooms, operations := fake.snapshot() - if len(rooms) != 0 { - t.Fatalf("rooms = %v, want none", rooms) - } - if !slices.Equal(operations, []string{"delete:" + current}) { - t.Fatalf("operations = %v", operations) - } -} - -func TestRoomControlRejectsForeignInstanceAndUnconfirmedDrain(t *testing.T) { - foreign, foreignURL := newFakeLiveKit(t) - foreign.seed("another-application") - err := testControl(t, foreignURL).Initialize(context.Background()) - if err == nil || !strings.Contains(err.Error(), "foreign room") { - t.Fatalf("Initialize error = %v, want a foreign room refusal", err) - } - if operations := foreign.operationLog(); len(operations) != 0 { - t.Fatalf("operations = %v, want none", operations) - } - - retained, retainedURL := newFakeLiveKit(t) - retained.retainDeletedRoom = true - retained.seed(ManagedRoomName(testFence())) - err = testControl(t, retainedURL).Initialize(context.Background()) - if err == nil || !strings.Contains(err.Error(), "not empty after startup drain") { - t.Fatalf("Initialize error = %v, want an unconfirmed drain", err) - } -} - -// TestRoomControlInitializeAcceptsAnEmptyNamespace pins the confirmation call -// itself: an already empty instance still asks a second time. -func TestRoomControlInitializeAcceptsAnEmptyNamespace(t *testing.T) { - fake, apiURL := newFakeLiveKit(t) - if err := testControl(t, apiURL).Initialize(context.Background()); err != nil { - t.Fatalf("Initialize: %v", err) - } - if operations := fake.operationLog(); len(operations) != 0 { - t.Fatalf("operations = %v, want none", operations) - } - fake.mu.Lock() - defer fake.mu.Unlock() - if fake.listCalls != 2 { - t.Fatalf("listRooms calls = %d, want the drain and its confirmation", fake.listCalls) - } -} - -func TestRoomControlRejectsOffLedgerRoom(t *testing.T) { - fake, apiURL := newFakeLiveKit(t) - fence := testFence() - fake.seed(ManagedRoomName(fence), "host") - - err := testControl(t, apiURL).CreateRoom(context.Background(), fence) - if err == nil || !strings.Contains(err.Error(), "already exists") { - t.Fatalf("CreateRoom error = %v, want an existing-room refusal", err) - } - if operations := fake.operationLog(); len(operations) != 0 { - t.Fatalf("operations = %v, want none", operations) - } -} - -func TestRoomControlChecksExactHostParticipant(t *testing.T) { - fake, apiURL := newFakeLiveKit(t) - control := testControl(t, apiURL) - fence := testFence() - ctx := context.Background() - if err := control.CreateRoom(ctx, fence); err != nil { - t.Fatalf("CreateRoom: %v", err) - } - roomName := ManagedRoomName(fence) - - if !fake.join(roomName, "viewer:one") { - t.Fatal("the viewer must join") - } - expectHost(t, control, fence, false) - if !fake.join(roomName, "host") { - t.Fatal("the host must join") - } - expectHost(t, control, fence, true) - if err := control.DeleteRoom(ctx, fence); err != nil { - t.Fatalf("DeleteRoom: %v", err) - } - // An absent room reads as an offline host, not as an error. - expectHost(t, control, fence, false) -} - -func TestRoomControlRemovesOnlyTheExactViewer(t *testing.T) { - fake, apiURL := newFakeLiveKit(t) - control := testControl(t, apiURL) - fence := testFence() - ctx := context.Background() - if err := control.CreateRoom(ctx, fence); err != nil { - t.Fatalf("CreateRoom: %v", err) - } - roomName := ManagedRoomName(fence) - const viewerPeerID = "viewer_peer_12345678" - for _, identity := range []string{"host", "viewer:" + viewerPeerID, "viewer:other_peer_12345678"} { - if !fake.join(roomName, identity) { - t.Fatalf("%s must join", identity) - } - } - - subscription := SubscriptionFence{ResourceFence: fence, ViewerPeerID: viewerPeerID} - if err := control.DrainSubscription(ctx, subscription); err != nil { - t.Fatalf("DrainSubscription: %v", err) - } - if participants := fake.participants(roomName); !slices.Equal(participants, - []string{"host", "viewer:other_peer_12345678"}) { - t.Fatalf("participants = %v", participants) - } - // The repeat drain is a not-found removal, which is tolerated. - if err := control.DrainSubscription(ctx, subscription); err != nil { - t.Fatalf("repeat DrainSubscription: %v", err) - } -} - -func TestRoomControlRejectsUnconfirmedViewerDrain(t *testing.T) { - fake, apiURL := newFakeLiveKit(t) - control := testControl(t, apiURL) - fence := testFence() - ctx := context.Background() - if err := control.CreateRoom(ctx, fence); err != nil { - t.Fatalf("CreateRoom: %v", err) - } - const viewerPeerID = "viewer_peer_12345678" - fake.join(ManagedRoomName(fence), "viewer:"+viewerPeerID) - fake.mu.Lock() - fake.retainRemovedParticipant = true - fake.mu.Unlock() - - err := control.DrainSubscription(ctx, - SubscriptionFence{ResourceFence: fence, ViewerPeerID: viewerPeerID}) - if err == nil || !strings.Contains(err.Error(), "removal was not confirmed") { - t.Fatalf("DrainSubscription error = %v, want an unconfirmed removal", err) - } -} - -func TestRoomControlRejectsInvalidConfiguration(t *testing.T) { - for name, options := range map[string]LiveKitRoomControlOptions{ - "no url": {APIURL: "", APIKey: testAPIKey, APISecret: testAPISecret, MaxViewersPerRoom: 8}, - "no key": {APIURL: "http://127.0.0.1:7880", APIKey: "", APISecret: testAPISecret, MaxViewersPerRoom: 8}, - "short secret": {APIURL: "http://127.0.0.1:7880", APIKey: testAPIKey, APISecret: strings.Repeat("s", 31), MaxViewersPerRoom: 8}, - "no viewers": {APIURL: "http://127.0.0.1:7880", APIKey: testAPIKey, APISecret: testAPISecret, MaxViewersPerRoom: 0}, - "too many viewers": { - APIURL: "http://127.0.0.1:7880", APIKey: testAPIKey, - APISecret: testAPISecret, MaxViewersPerRoom: 21, - }, - } { - if _, err := NewLiveKitRoomControl(options); err == nil { - t.Fatalf("%s: want a rejection", name) - } - } - - // An invalid fence is refused without touching the service. - fake, apiURL := newFakeLiveKit(t) - control := testControl(t, apiURL) - broken := ResourceFence{RoomID: "0", ShareGeneration: testShareGeneration, - PublicationGeneration: "publication_generation_12345678"} - if err := control.CreateRoom(context.Background(), broken); err == nil { - t.Fatal("an invalid fence must be refused") - } - if err := control.DrainSubscription(context.Background(), SubscriptionFence{ - ResourceFence: testFence(), ViewerPeerID: "short", - }); err == nil { - t.Fatal("an invalid viewer identity must be refused") - } - if operations := fake.operationLog(); len(operations) != 0 { - t.Fatalf("operations = %v, want none", operations) - } -} - -// TestRoomControlPropagatesContextCancellation covers the ctx plumbing that -// carries the 5 second per-request timeout. -func TestRoomControlPropagatesContextCancellation(t *testing.T) { - _, apiURL := newFakeLiveKit(t) - control := testControl(t, apiURL) - ctx, cancel := context.WithCancel(context.Background()) - cancel() - if err := control.CreateRoom(ctx, testFence()); err == nil { - t.Fatal("a cancelled context must abort the call") - } -} - -// TestRoomControlAutoCreateStaysDisabled keeps the deployment contract the -// TypeScript suite asserted next to these scenarios. -func TestRoomControlAutoCreateStaysDisabled(t *testing.T) { - configuration, err := os.ReadFile("../../../deploy/livekit/livekit.yaml.example") - if err != nil { - t.Fatalf("read livekit.yaml.example: %v", err) - } - if !regexp.MustCompile(`(?s)room:\s+.*auto_create: false`).Match(configuration) { - t.Fatal("the tracked LiveKit deployment must keep auto_create disabled") - } -} - -func expectHost(t *testing.T, control *LiveKitRoomControl, fence ResourceFence, want bool) { - t.Helper() - exists, err := control.HostParticipantExists(context.Background(), fence) - if err != nil { - t.Fatalf("HostParticipantExists: %v", err) - } - if exists != want { - t.Fatalf("HostParticipantExists = %v, want %v", exists, want) - } -} diff --git a/internal/server/sfu/roomservice.go b/internal/server/sfu/roomservice.go deleted file mode 100644 index 87657d9..0000000 --- a/internal/server/sfu/roomservice.go +++ /dev/null @@ -1,188 +0,0 @@ -package sfu - -// Twirp/JSON client for livekit.RoomService, covering exactly the five RPCs -// src/server/sfu-room-control.ts used through RoomServiceClient. The wire -// contract is evidence/livekit.md section 3. - -import ( - "bytes" - "context" - "encoding/json" - "errors" - "fmt" - "io" - "net/http" - "strings" - "time" -) - -const ( - // requestTimeout is RoomServiceClient's `requestTimeout: 5`, applied per - // HTTP request. There is no retry and no failover. - requestTimeout = 5 * time.Second - twirpPathPrefix = "/twirp/livekit.RoomService/" - // maxResponseBytes bounds a hostile or broken RoomService reply; the four - // decoded shapes are tiny. - maxResponseBytes = 1 << 20 -) - -// twirpError is the Twirp JSON error body {"code":..,"msg":..} plus the HTTP -// status. It is unexported on purpose: the only question the server ever asks -// is isNotFound, and the body must never be logged. -type twirpError struct { - status int - code string - message string -} - -func (failure *twirpError) Error() string { - return fmt.Sprintf("livekit %s: %s (HTTP %d)", - failure.code, failure.message, failure.status) -} - -// isNotFound ports isNotFound(): HTTP 404 or the twirp code "not_found", and -// nothing else, so a 409 already_exists still propagates. -func isNotFound(err error) bool { - var failure *twirpError - return errors.As(err, &failure) && - (failure.status == http.StatusNotFound || failure.code == "not_found") -} - -type roomRecord struct { - Name string `json:"name"` -} - -type participantRecord struct { - Identity string `json:"identity"` -} - -// roomService is the LiveKitRoomService interface of the TypeScript, bound to -// the real RoomService rather than to an injected double. -type roomService struct { - baseURL string - signer signer -} - -// newRoomService applies TwirpRpc's ws->http host rewrite and drops a trailing -// slash so the path prefix below is exact. -func newRoomService(apiURL string, credentials signer) roomService { - if strings.HasPrefix(apiURL, "ws") { - apiURL = "http" + strings.TrimPrefix(apiURL, "ws") - } - return roomService{baseURL: strings.TrimSuffix(apiURL, "/"), signer: credentials} -} - -func (service roomService) call( - ctx context.Context, - method string, - grant videoGrant, - request any, - response any, -) error { - body, err := json.Marshal(request) - if err != nil { - return err - } - token, err := service.signer.serviceToken(grant) - if err != nil { - return err - } - ctx, cancel := context.WithTimeout(ctx, requestTimeout) - defer cancel() - httpRequest, err := http.NewRequestWithContext(ctx, http.MethodPost, - service.baseURL+twirpPathPrefix+method, bytes.NewReader(body)) - if err != nil { - return err - } - httpRequest.Header.Set("Content-Type", "application/json;charset=UTF-8") - httpRequest.Header.Set("Authorization", "Bearer "+token) - httpResponse, err := http.DefaultClient.Do(httpRequest) - if err != nil { - return err - } - defer httpResponse.Body.Close() - payload, err := io.ReadAll(io.LimitReader(httpResponse.Body, maxResponseBytes)) - if err != nil { - return err - } - if httpResponse.StatusCode < 200 || httpResponse.StatusCode >= 300 { - return newTwirpError(httpResponse, payload) - } - if response == nil { - return nil - } - return json.Unmarshal(payload, response) -} - -// newTwirpError decodes {"code","msg"} when the reply claims JSON. LiveKit's -// own 401 is plain text or {"Error":...}, which keeps code empty and so is not -// a not-found, exactly as the TypeScript SDK behaved. -func newTwirpError(response *http.Response, payload []byte) *twirpError { - failure := &twirpError{status: response.StatusCode, message: "Unknown internal error"} - if strings.HasPrefix(response.Header.Get("Content-Type"), "application/json") { - var body struct { - Code string `json:"code"` - Msg string `json:"msg"` - } - if json.Unmarshal(payload, &body) == nil && body.Code != "" { - failure.code = body.Code - failure.message = body.Msg - } - } - return failure -} - -func (service roomService) listRooms(ctx context.Context, names []string) ([]roomRecord, error) { - // protobuf-es omits an empty repeated field, so an unfiltered list sends {}. - request := struct { - Names []string `json:"names,omitempty"` - }{Names: names} - var response struct { - Rooms []roomRecord `json:"rooms"` - } - if err := service.call(ctx, "ListRooms", - videoGrant{RoomList: true}, request, &response); err != nil { - return nil, err - } - return response.Rooms, nil -} - -func (service roomService) createRoom(ctx context.Context, room string, maxParticipants int) error { - request := struct { - Name string `json:"name"` - MaxParticipants int `json:"maxParticipants"` - }{Name: room, MaxParticipants: maxParticipants} - // The TypeScript discarded the returned Room record. - return service.call(ctx, "CreateRoom", videoGrant{RoomCreate: true}, request, nil) -} - -func (service roomService) deleteRoom(ctx context.Context, room string) error { - request := struct { - Room string `json:"room"` - }{Room: room} - return service.call(ctx, "DeleteRoom", videoGrant{RoomCreate: true}, request, nil) -} - -func (service roomService) listParticipants(ctx context.Context, room string) ([]participantRecord, error) { - request := struct { - Room string `json:"room"` - }{Room: room} - var response struct { - Participants []participantRecord `json:"participants"` - } - if err := service.call(ctx, "ListParticipants", - videoGrant{RoomAdmin: true, Room: room}, request, &response); err != nil { - return nil, err - } - return response.Participants, nil -} - -func (service roomService) removeParticipant(ctx context.Context, room string, identity string) error { - // revokeTokenTs stays absent, as the TypeScript passed undefined. - request := struct { - Room string `json:"room"` - Identity string `json:"identity"` - }{Room: room, Identity: identity} - return service.call(ctx, "RemoveParticipant", - videoGrant{RoomAdmin: true, Room: room}, request, nil) -} diff --git a/internal/server/sfu/roomservice_test.go b/internal/server/sfu/roomservice_test.go deleted file mode 100644 index 3f6c45c..0000000 --- a/internal/server/sfu/roomservice_test.go +++ /dev/null @@ -1,178 +0,0 @@ -package sfu - -// The Twirp wire contract recorded in evidence/livekit.md section 3: request -// path, method, headers, body and per-call service-token grant for each of the -// five RPCs, plus the exact not-found rule. - -import ( - "context" - "encoding/json" - "net/http" - "net/http/httptest" - "reflect" - "strings" - "testing" -) - -type capturedCall struct { - path string - method string - contentType string - authorized string - body map[string]any -} - -func captureLiveKit( - t *testing.T, - reply func(method string) (int, string, string), -) (*[]capturedCall, string) { - t.Helper() - var calls []capturedCall - server := httptest.NewServer(http.HandlerFunc( - func(writer http.ResponseWriter, request *http.Request) { - var body map[string]any - _ = json.NewDecoder(request.Body).Decode(&body) - if body == nil { - body = map[string]any{} - } - calls = append(calls, capturedCall{ - path: request.URL.Path, - method: request.Method, - contentType: request.Header.Get("Content-Type"), - authorized: request.Header.Get("Authorization"), - body: body, - }) - name := request.URL.Path[strings.LastIndexByte(request.URL.Path, '/')+1:] - status, contentType, payload := reply(name) - writer.Header().Set("Content-Type", contentType) - writer.WriteHeader(status) - _, _ = writer.Write([]byte(payload)) - })) - t.Cleanup(server.Close) - return &calls, server.URL -} - -func TestRoomServiceRequestShape(t *testing.T) { - const room = "screener-v1.42.a_1234567.b_1234567" - calls, apiURL := captureLiveKit(t, func(method string) (int, string, string) { - switch method { - case "ListRooms": - return 200, "application/json", `{"rooms":[{"sid":"RM_x","name":"` + room + `","empty_timeout":0}]}` - case "ListParticipants": - return 200, "application/json", `{"participants":[{"sid":"PA_x","identity":"host","state":"ACTIVE"}]}` - default: - return 200, "application/json", `{}` - } - }) - // The ws->http rewrite of TwirpRpc, exercised on the way in. - service := newRoomService(strings.Replace(apiURL, "http", "ws", 1), - newSigner(testAPIKey, testAPISecret)) - ctx := context.Background() - - if _, err := service.listRooms(ctx, nil); err != nil { - t.Fatalf("listRooms: %v", err) - } - if _, err := service.listRooms(ctx, []string{room}); err != nil { - t.Fatalf("filtered listRooms: %v", err) - } - if err := service.createRoom(ctx, room, 9); err != nil { - t.Fatalf("createRoom: %v", err) - } - if err := service.deleteRoom(ctx, room); err != nil { - t.Fatalf("deleteRoom: %v", err) - } - participants, err := service.listParticipants(ctx, room) - if err != nil { - t.Fatalf("listParticipants: %v", err) - } - if len(participants) != 1 || participants[0].Identity != "host" { - t.Fatalf("participants = %v", participants) - } - if err := service.removeParticipant(ctx, room, "viewer:viewer_peer_12345678"); err != nil { - t.Fatalf("removeParticipant: %v", err) - } - - wanted := []struct { - path string - body map[string]any - grant map[string]any - }{ - {"ListRooms", map[string]any{}, map[string]any{"roomList": true}}, - {"ListRooms", map[string]any{"names": []any{room}}, map[string]any{"roomList": true}}, - {"CreateRoom", map[string]any{"name": room, "maxParticipants": float64(9)}, - map[string]any{"roomCreate": true}}, - {"DeleteRoom", map[string]any{"room": room}, map[string]any{"roomCreate": true}}, - {"ListParticipants", map[string]any{"room": room}, - map[string]any{"roomAdmin": true, "room": room}}, - {"RemoveParticipant", map[string]any{"room": room, "identity": "viewer:viewer_peer_12345678"}, - map[string]any{"roomAdmin": true, "room": room}}, - } - if len(*calls) != len(wanted) { - t.Fatalf("calls = %d, want %d", len(*calls), len(wanted)) - } - for index, call := range *calls { - want := wanted[index] - if call.path != "/twirp/livekit.RoomService/"+want.path { - t.Fatalf("call %d path = %s", index, call.path) - } - if call.method != http.MethodPost { - t.Fatalf("call %d method = %s", index, call.method) - } - if call.contentType != "application/json;charset=UTF-8" { - t.Fatalf("call %d content-type = %q", index, call.contentType) - } - if !strings.HasPrefix(call.authorized, "Bearer ") { - t.Fatalf("call %d authorization = %q", index, call.authorized) - } - if !reflect.DeepEqual(call.body, want.body) { - t.Fatalf("call %d body = %v, want %v", index, call.body, want.body) - } - - claims := decodeSegment(t, - strings.Split(strings.TrimPrefix(call.authorized, "Bearer "), ".")[1]) - if _, present := claims["sub"]; present { - t.Fatalf("call %d service token must carry no sub", index) - } - if claims["iss"] != testAPIKey { - t.Fatalf("call %d iss = %v", index, claims["iss"]) - } - if lifetime := claims["exp"].(float64) - claims["nbf"].(float64); lifetime != 600 { - t.Fatalf("call %d service token exp - nbf = %v, want 600", index, lifetime) - } - grant, _ := claims["video"].(map[string]any) - if !reflect.DeepEqual(grant, want.grant) { - t.Fatalf("call %d grant = %v, want %v", index, grant, want.grant) - } - } -} - -// TestRoomServiceNotFoundRule pins isNotFound: HTTP 404 or code "not_found", -// nothing else. Both arms of the TypeScript `||` are reachable. -func TestRoomServiceNotFoundRule(t *testing.T) { - for name, testCase := range map[string]struct { - status int - contentType string - payload string - notFound bool - }{ - "twirp not found": {404, "application/json", `{"code":"not_found","msg":"missing"}`, true}, - "status only": {404, "text/plain", "Not Found", true}, - "code only": {500, "application/json", `{"code":"not_found","msg":"missing"}`, true}, - "already exists": {409, "application/json", `{"code":"already_exists","msg":"duplicate"}`, false}, - "internal": {500, "application/json", `{"code":"internal","msg":"boom"}`, false}, - "unauthenticated": {401, "application/json", `{"Error":"invalid API key"}`, false}, - "plain text error": {503, "text/plain", "unavailable", false}, - } { - _, apiURL := captureLiveKit(t, func(string) (int, string, string) { - return testCase.status, testCase.contentType, testCase.payload - }) - service := newRoomService(apiURL, newSigner(testAPIKey, testAPISecret)) - err := service.deleteRoom(context.Background(), "screener-v1.42.a_1234567.b_1234567") - if err == nil { - t.Fatalf("%s: want an error", name) - } - if isNotFound(err) != testCase.notFound { - t.Fatalf("%s: isNotFound = %v, want %v (%v)", name, !testCase.notFound, testCase.notFound, err) - } - } -} diff --git a/internal/server/sfu/sfutest/fake.go b/internal/server/sfu/sfutest/fake.go index 19bbfee..0c244bd 100644 --- a/internal/server/sfu/sfutest/fake.go +++ b/internal/server/sfu/sfutest/fake.go @@ -1,271 +1,115 @@ -// Package sfutest holds the SFU test double the TypeScript suite kept in -// tests/fake-sfu-room-control.ts, so the signaling, router and app tests can -// drive the LiveKit fallback without a LiveKit. +// Package sfutest provides deterministic physical-media effects for router tests. package sfutest import ( "context" "errors" - "sort" + "slices" "sync" - "github.com/TNTcraftHIM/Screener/internal/server/ordered" "github.com/TNTcraftHIM/Screener/internal/server/sfu" + "github.com/pion/webrtc/v4" ) -// FakeRoomControl is FakeSfuRoomControl. Every field is guarded: unlike the -// single-threaded TypeScript, the router calls it from goroutines while the -// test asserts. A barrier is a channel the test closes to release the call; -// a nil barrier does not block, which is the TypeScript `await undefined`. -type FakeRoomControl struct { - mu sync.Mutex - // Ordering site: initialize reports the room names in insertion order. - rooms ordered.Map[string, map[string]struct{}] - created []sfu.ResourceFence - deleted []sfu.ResourceFence - subscriptionDrainAttempts []sfu.SubscriptionFence - drainedSubscriptions []sfu.SubscriptionFence - startupDeletedRoomNames []string - initializeCalls int - initializeBarrier chan struct{} - initializeError error - createBarrier chan struct{} - subscriptionDrainBarrier chan struct{} - failDelete bool - failHostCheck bool +type FakeMedia struct { + mu sync.Mutex + publications map[sfu.ResourceFence]string + created []sfu.ResourceFence + deleted []sfu.ResourceFence + drained []sfu.SubscriptionFence + drainAttempts []sfu.SubscriptionFence + deleteErr error + drainBarrier <-chan struct{} } -var _ sfu.RoomControl = (*FakeRoomControl)(nil) - -// New returns an empty fake. -func New() *FakeRoomControl { return &FakeRoomControl{} } - -// Initialize records the call, then clears every room. -func (fake *FakeRoomControl) Initialize(ctx context.Context) error { - fake.mu.Lock() - fake.initializeCalls++ - barrier := fake.initializeBarrier - fake.mu.Unlock() - - if err := wait(ctx, barrier); err != nil { - return err - } - - fake.mu.Lock() - defer fake.mu.Unlock() - if fake.initializeError != nil { - return fake.initializeError - } - fake.startupDeletedRoomNames = append(fake.startupDeletedRoomNames, fake.rooms.Keys()...) - fake.rooms.Clear() +func New() *FakeMedia { return &FakeMedia{publications: make(map[sfu.ResourceFence]string)} } +func (media *FakeMedia) HasPublication(fence sfu.ResourceFence, connectionID string) bool { + media.mu.Lock() + defer media.mu.Unlock() + return media.publications[fence] == connectionID && connectionID != "" +} +func (media *FakeMedia) PreparePublication(fence sfu.ResourceFence, connectionID string, _ sfu.PublicationMedia) error { + media.mu.Lock() + defer media.mu.Unlock() + media.publications[fence] = connectionID + media.created = append(media.created, fence) return nil } - -// CreateRoom refuses a room name that already exists. -func (fake *FakeRoomControl) CreateRoom(ctx context.Context, fence sfu.ResourceFence) error { - fake.mu.Lock() - barrier := fake.createBarrier - fake.mu.Unlock() - - if err := wait(ctx, barrier); err != nil { - return err - } - roomName := sfu.ManagedRoomName(fence) - - fake.mu.Lock() - defer fake.mu.Unlock() - if fake.rooms.Has(roomName) { - return errors.New("room already exists") - } - fake.rooms.Set(roomName, map[string]struct{}{}) - fake.created = append(fake.created, fence) +func (*FakeMedia) AcceptPublisherOffer(sfu.ResourceFence, string, webrtc.SessionDescription) (webrtc.SessionDescription, error) { + return webrtc.SessionDescription{Type: webrtc.SDPTypeAnswer, SDP: "answer"}, nil +} +func (*FakeMedia) AddPublisherICE(sfu.ResourceFence, string, webrtc.ICECandidateInit) error { return nil } - -// DeleteRoom removes the room, whether or not it existed. -func (fake *FakeRoomControl) DeleteRoom(ctx context.Context, fence sfu.ResourceFence) error { - fake.mu.Lock() - defer fake.mu.Unlock() - if fake.failDelete { - return errors.New("room deletion failed") - } - fake.rooms.Delete(sfu.ManagedRoomName(fence)) - fake.deleted = append(fake.deleted, fence) +func (*FakeMedia) UpdatePublication(sfu.ResourceFence, string, sfu.PublicationMedia) error { return nil } - -// DrainSubscription records the attempt before waiting, then removes the viewer. -func (fake *FakeRoomControl) DrainSubscription(ctx context.Context, fence sfu.SubscriptionFence) error { - fake.mu.Lock() - fake.subscriptionDrainAttempts = append(fake.subscriptionDrainAttempts, fence) - barrier := fake.subscriptionDrainBarrier - fake.mu.Unlock() - - if err := wait(ctx, barrier); err != nil { - return err - } - - fake.mu.Lock() - defer fake.mu.Unlock() - if participants, ok := fake.rooms.Get(sfu.ManagedRoomName(fence.ResourceFence)); ok { - delete(participants, "viewer:"+fence.ViewerPeerID) - } - fake.drainedSubscriptions = append(fake.drainedSubscriptions, fence) +func (*FakeMedia) PublicationDemand(sfu.ResourceFence, string) (int, error) { return 1, nil } +func (*FakeMedia) PrepareSubscriber(context.Context, sfu.SubscriptionFence, string) (webrtc.SessionDescription, error) { + return webrtc.SessionDescription{Type: webrtc.SDPTypeOffer, SDP: "offer"}, nil +} +func (*FakeMedia) ApplySubscriberAnswer(sfu.SubscriptionFence, string, webrtc.SessionDescription) error { return nil } - -// HostParticipantExists reports whether "host" joined the fenced room. -func (fake *FakeRoomControl) HostParticipantExists( - _ context.Context, - fence sfu.ResourceFence, -) (bool, error) { - fake.mu.Lock() - defer fake.mu.Unlock() - if fake.failHostCheck { - return false, errors.New("host check failed") +func (*FakeMedia) AddSubscriberICE(sfu.SubscriptionFence, string, webrtc.ICECandidateInit) error { + return nil +} +func (*FakeMedia) RestartSubscriber(sfu.SubscriptionFence, string) (webrtc.SessionDescription, error) { + return webrtc.SessionDescription{Type: webrtc.SDPTypeOffer, SDP: "restart"}, nil +} +func (media *FakeMedia) CloseSubscription(fence sfu.SubscriptionFence, _ string) error { + media.mu.Lock() + media.drainAttempts = append(media.drainAttempts, fence) + barrier := media.drainBarrier + media.mu.Unlock() + if barrier != nil { + <-barrier } - participants, ok := fake.rooms.Get(sfu.ManagedRoomName(fence)) - if !ok { - return false, nil + media.mu.Lock() + defer media.mu.Unlock() + media.drained = append(media.drained, fence) + return nil +} +func (media *FakeMedia) ClosePublication(fence sfu.ResourceFence) error { + media.mu.Lock() + defer media.mu.Unlock() + if media.deleteErr != nil { + return media.deleteErr } - _, joined := participants["host"] - return joined, nil + delete(media.publications, fence) + media.deleted = append(media.deleted, fence) + return nil } - -// SeedRoom creates the fenced room with the given identities present. -func (fake *FakeRoomControl) SeedRoom(fence sfu.ResourceFence, identities ...string) { - participants := make(map[string]struct{}, len(identities)) - for _, identity := range identities { - participants[identity] = struct{}{} - } - fake.mu.Lock() - defer fake.mu.Unlock() - fake.rooms.Set(sfu.ManagedRoomName(fence), participants) -} - -// CanJoin reports whether the room name exists. -func (fake *FakeRoomControl) CanJoin(roomName string) bool { - fake.mu.Lock() - defer fake.mu.Unlock() - return fake.rooms.Has(roomName) -} - -// RoomCount is rooms.size. -func (fake *FakeRoomControl) RoomCount() int { - fake.mu.Lock() - defer fake.mu.Unlock() - return fake.rooms.Len() -} - -// Participants lists the identities in a room, sorted for a stable assertion. -func (fake *FakeRoomControl) Participants(roomName string) []string { - fake.mu.Lock() - defer fake.mu.Unlock() - participants, ok := fake.rooms.Get(roomName) - if !ok { - return nil - } - identities := make([]string, 0, len(participants)) - for identity := range participants { - identities = append(identities, identity) - } - sort.Strings(identities) - return identities -} - -// InitializeCalls is initializeCalls. -func (fake *FakeRoomControl) InitializeCalls() int { - fake.mu.Lock() - defer fake.mu.Unlock() - return fake.initializeCalls -} - -// StartupDeletedRoomNames is startupDeletedRoomNames. -func (fake *FakeRoomControl) StartupDeletedRoomNames() []string { - fake.mu.Lock() - defer fake.mu.Unlock() - return append([]string(nil), fake.startupDeletedRoomNames...) -} - -// Created is created. -func (fake *FakeRoomControl) Created() []sfu.ResourceFence { - fake.mu.Lock() - defer fake.mu.Unlock() - return append([]sfu.ResourceFence(nil), fake.created...) -} - -// Deleted is deleted. -func (fake *FakeRoomControl) Deleted() []sfu.ResourceFence { - fake.mu.Lock() - defer fake.mu.Unlock() - return append([]sfu.ResourceFence(nil), fake.deleted...) -} - -// SubscriptionDrainAttempts is subscriptionDrainAttempts. -func (fake *FakeRoomControl) SubscriptionDrainAttempts() []sfu.SubscriptionFence { - fake.mu.Lock() - defer fake.mu.Unlock() - return append([]sfu.SubscriptionFence(nil), fake.subscriptionDrainAttempts...) -} - -// DrainedSubscriptions is drainedSubscriptions. -func (fake *FakeRoomControl) DrainedSubscriptions() []sfu.SubscriptionFence { - fake.mu.Lock() - defer fake.mu.Unlock() - return append([]sfu.SubscriptionFence(nil), fake.drainedSubscriptions...) -} - -// SetInitializeBarrier blocks Initialize until barrier is closed. -func (fake *FakeRoomControl) SetInitializeBarrier(barrier chan struct{}) { - fake.mu.Lock() - defer fake.mu.Unlock() - fake.initializeBarrier = barrier -} - -// SetInitializeError makes Initialize fail after its barrier. -func (fake *FakeRoomControl) SetInitializeError(err error) { - fake.mu.Lock() - defer fake.mu.Unlock() - fake.initializeError = err -} - -// SetCreateBarrier blocks CreateRoom until barrier is closed. -func (fake *FakeRoomControl) SetCreateBarrier(barrier chan struct{}) { - fake.mu.Lock() - defer fake.mu.Unlock() - fake.createBarrier = barrier -} - -// SetSubscriptionDrainBarrier blocks DrainSubscription until barrier is closed. -func (fake *FakeRoomControl) SetSubscriptionDrainBarrier(barrier chan struct{}) { - fake.mu.Lock() - defer fake.mu.Unlock() - fake.subscriptionDrainBarrier = barrier -} - -// SetFailDelete makes DeleteRoom fail. -func (fake *FakeRoomControl) SetFailDelete(fail bool) { - fake.mu.Lock() - defer fake.mu.Unlock() - fake.failDelete = fail -} - -// SetFailHostCheck makes HostParticipantExists fail. -func (fake *FakeRoomControl) SetFailHostCheck(fail bool) { - fake.mu.Lock() - defer fake.mu.Unlock() - fake.failHostCheck = fail -} - -// wait is `await barrier` with the Go escape hatch a cancelled context needs. -func wait(ctx context.Context, barrier chan struct{}) error { - if barrier == nil { - return nil - } - select { - case <-barrier: - return nil - case <-ctx.Done(): - return ctx.Err() +func (media *FakeMedia) SetFailDelete(fail bool) { + media.mu.Lock() + defer media.mu.Unlock() + media.deleteErr = nil + if fail { + media.deleteErr = errors.New("test media close failed") } } +func (media *FakeMedia) SetSubscriptionDrainBarrier(barrier <-chan struct{}) { + media.mu.Lock() + defer media.mu.Unlock() + media.drainBarrier = barrier +} +func (media *FakeMedia) Created() []sfu.ResourceFence { + media.mu.Lock() + defer media.mu.Unlock() + return slices.Clone(media.created) +} +func (media *FakeMedia) Deleted() []sfu.ResourceFence { + media.mu.Lock() + defer media.mu.Unlock() + return slices.Clone(media.deleted) +} +func (media *FakeMedia) DrainedSubscriptions() []sfu.SubscriptionFence { + media.mu.Lock() + defer media.mu.Unlock() + return slices.Clone(media.drained) +} +func (media *FakeMedia) SubscriptionDrainAttempts() []sfu.SubscriptionFence { + media.mu.Lock() + defer media.mu.Unlock() + return slices.Clone(media.drainAttempts) +} diff --git a/internal/server/sfu/sfutest/fake_test.go b/internal/server/sfu/sfutest/fake_test.go deleted file mode 100644 index 836223a..0000000 --- a/internal/server/sfu/sfutest/fake_test.go +++ /dev/null @@ -1,193 +0,0 @@ -package sfutest - -// The behaviour tests/fake-sfu-room-control.ts guarantees to the scenarios that -// use it, so a signaling or app test can rely on it the same way. - -import ( - "context" - "errors" - "slices" - "testing" - "time" - - "github.com/TNTcraftHIM/Screener/internal/server/sfu" -) - -func fence(publicationGeneration string) sfu.ResourceFence { - return sfu.ResourceFence{ - RoomID: "42", - ShareGeneration: "share_generation_12345678", - PublicationGeneration: publicationGeneration, - } -} - -func TestFakeInitializeDrainsSeededRooms(t *testing.T) { - fake := New() - first := fence("publication_generation_12345678") - second := fence("publication_generation_87654321") - fake.SeedRoom(first, "host") - fake.SeedRoom(second) - barrier := make(chan struct{}) - fake.SetInitializeBarrier(barrier) - - done := make(chan error, 1) - go func() { done <- fake.Initialize(context.Background()) }() - // The call is counted before the barrier, which is what a startup test that - // asserts "initialize was entered" observes. - waitFor(t, "the initialize call", func() bool { return fake.InitializeCalls() == 1 }) - if fake.RoomCount() != 2 { - t.Fatalf("rooms = %d, want the seeded pair", fake.RoomCount()) - } - - close(barrier) - if err := <-done; err != nil { - t.Fatalf("Initialize: %v", err) - } - if names := fake.StartupDeletedRoomNames(); !slices.Equal(names, - []string{sfu.ManagedRoomName(first), sfu.ManagedRoomName(second)}) { - t.Fatalf("startup deleted = %v, want both rooms in seeding order", names) - } - if fake.RoomCount() != 0 { - t.Fatalf("rooms = %d, want none", fake.RoomCount()) - } -} - -func TestFakeInitializeReportsInjectedFailure(t *testing.T) { - fake := New() - failure := errors.New("startup reconciliation failed") - fake.SetInitializeError(failure) - fake.SeedRoom(fence("publication_generation_12345678")) - - if err := fake.Initialize(context.Background()); !errors.Is(err, failure) { - t.Fatalf("Initialize error = %v, want %v", err, failure) - } - if fake.RoomCount() != 1 { - t.Fatal("a failed initialize must not clear the rooms") - } -} - -func TestFakeRecordsRoomLifecycle(t *testing.T) { - fake := New() - ctx := context.Background() - current := fence("publication_generation_12345678") - roomName := sfu.ManagedRoomName(current) - - if err := fake.CreateRoom(ctx, current); err != nil { - t.Fatalf("CreateRoom: %v", err) - } - if err := fake.CreateRoom(ctx, current); err == nil { - t.Fatal("a duplicate room must be refused") - } - if !fake.CanJoin(roomName) { - t.Fatal("the created room must accept a join") - } - if created := fake.Created(); !slices.Equal(created, []sfu.ResourceFence{current}) { - t.Fatalf("created = %v", created) - } - - fake.SetFailDelete(true) - if err := fake.DeleteRoom(ctx, current); err == nil { - t.Fatal("the injected delete failure must surface") - } - if !fake.CanJoin(roomName) || len(fake.Deleted()) != 0 { - t.Fatal("a failed delete must change nothing") - } - fake.SetFailDelete(false) - if err := fake.DeleteRoom(ctx, current); err != nil { - t.Fatalf("DeleteRoom: %v", err) - } - if fake.CanJoin(roomName) { - t.Fatal("the deleted room must not accept a join") - } - if deleted := fake.Deleted(); !slices.Equal(deleted, []sfu.ResourceFence{current}) { - t.Fatalf("deleted = %v", deleted) - } -} - -func TestFakeDrainsOnlyTheExactViewer(t *testing.T) { - fake := New() - ctx := context.Background() - current := fence("publication_generation_12345678") - roomName := sfu.ManagedRoomName(current) - fake.SeedRoom(current, "host", "viewer:viewer_peer_12345678", "viewer:other_peer_12345678") - barrier := make(chan struct{}) - fake.SetSubscriptionDrainBarrier(barrier) - - subscription := sfu.SubscriptionFence{ - ResourceFence: current, - ViewerPeerID: "viewer_peer_12345678", - } - done := make(chan error, 1) - go func() { done <- fake.DrainSubscription(ctx, subscription) }() - // The attempt is recorded before the barrier; the removal is not. - waitFor(t, "the drain attempt", func() bool { - return len(fake.SubscriptionDrainAttempts()) == 1 - }) - if len(fake.DrainedSubscriptions()) != 0 { - t.Fatal("the drain must not complete before its barrier") - } - - close(barrier) - if err := <-done; err != nil { - t.Fatalf("DrainSubscription: %v", err) - } - if participants := fake.Participants(roomName); !slices.Equal(participants, - []string{"host", "viewer:other_peer_12345678"}) { - t.Fatalf("participants = %v", participants) - } - if drained := fake.DrainedSubscriptions(); !slices.Equal(drained, - []sfu.SubscriptionFence{subscription}) { - t.Fatalf("drained = %v", drained) - } -} - -func TestFakeHostParticipantExists(t *testing.T) { - fake := New() - ctx := context.Background() - hosted := fence("publication_generation_12345678") - empty := fence("publication_generation_87654321") - fake.SeedRoom(hosted, "host") - fake.SeedRoom(empty, "viewer:viewer_peer_12345678") - - for _, testCase := range []struct { - fence sfu.ResourceFence - want bool - }{{hosted, true}, {empty, false}, {fence("publication_generation_11112222"), false}} { - exists, err := fake.HostParticipantExists(ctx, testCase.fence) - if err != nil { - t.Fatalf("HostParticipantExists: %v", err) - } - if exists != testCase.want { - t.Fatalf("HostParticipantExists(%s) = %v, want %v", - testCase.fence.PublicationGeneration, exists, testCase.want) - } - } - - fake.SetFailHostCheck(true) - if _, err := fake.HostParticipantExists(ctx, hosted); err == nil { - t.Fatal("the injected host-check failure must surface") - } -} - -func TestFakeReleasesBlockedCallsOnCancellation(t *testing.T) { - fake := New() - fake.SetCreateBarrier(make(chan struct{})) - ctx, cancel := context.WithCancel(context.Background()) - cancel() - if err := fake.CreateRoom(ctx, fence("publication_generation_12345678")); err == nil { - t.Fatal("a cancelled context must release the barrier") - } -} - -// waitFor is vitest's vi.waitFor: poll until the condition holds. -func waitFor(t *testing.T, description string, condition func() bool) { - t.Helper() - deadline := time.Now().Add(5 * time.Second) - for time.Now().Before(deadline) { - if condition() { - return - } - time.Sleep(time.Millisecond) - } - t.Fatalf("timed out waiting for %s", description) -} diff --git a/internal/server/sfu/token.go b/internal/server/sfu/token.go deleted file mode 100644 index 650d6fd..0000000 --- a/internal/server/sfu/token.go +++ /dev/null @@ -1,203 +0,0 @@ -package sfu - -// Ported from src/server/livekit-token.ts. The AccessToken of -// livekit-server-sdk is replaced by the HS256 JWT it produced; the claim set is -// the one recorded in evidence/livekit.md section 2. - -import ( - "crypto/hmac" - "crypto/sha256" - "encoding/base64" - "encoding/json" - "errors" - "time" - - "github.com/TNTcraftHIM/Screener/internal/server/protocol" -) - -const ( - // tokenTTLSeconds and serviceTokenTTLSeconds are the AccessToken `ttl` - // values of src/server/livekit-token.ts and the SDK's ServiceBase: a - // participant token only has to survive the join it was issued for, and a - // service token only the one RoomService call it authorises. - tokenTTLSeconds = 5 * 60 - serviceTokenTTLSeconds = 10 * 60 - // minAPISecretBytes is the TypeScript Buffer.byteLength(secret) >= 32 guard, - // which is len() in Go because both count UTF-8 bytes. - minAPISecretBytes = 32 - // TrackSource enum strings, from livekit-server-sdk grants.ts - // trackSourceToString(). - trackSourceScreenShare = "screen_share" - trackSourceScreenShareAudio = "screen_share_audio" -) - -// videoGrant is the subset of the SDK's VideoGrant this server sets. The four -// capability flags are *bool because absent and false differ to LiveKit: -// GetCanPublish() answers true for an absent flag, so a viewer's -// "canPublish": false must be emitted, not omitted. -type videoGrant struct { - RoomCreate bool `json:"roomCreate,omitempty"` - RoomJoin bool `json:"roomJoin,omitempty"` - RoomList bool `json:"roomList,omitempty"` - RoomAdmin bool `json:"roomAdmin,omitempty"` - Room string `json:"room,omitempty"` - CanPublish *bool `json:"canPublish,omitempty"` - CanPublishSources []string `json:"canPublishSources,omitempty"` - CanSubscribe *bool `json:"canSubscribe,omitempty"` - CanPublishData *bool `json:"canPublishData,omitempty"` - CanUpdateOwnMetadata *bool `json:"canUpdateOwnMetadata,omitempty"` -} - -// tokenClaims is the registered claim set jose emitted for AccessToken: no -// `iat`, no duplicate `identity`, and no `sub` on a service token. -type tokenClaims struct { - Iss string `json:"iss"` - Sub string `json:"sub,omitempty"` - Nbf int64 `json:"nbf"` - Exp int64 `json:"exp"` - Video videoGrant `json:"video"` -} - -// signer holds the API credentials shared by the participant token issuer and -// the RoomService client. -type signer struct { - apiKey string - apiSecret []byte -} - -func newSigner(apiKey string, apiSecret string) signer { - return signer{apiKey: apiKey, apiSecret: []byte(apiSecret)} -} - -// sign emits the compact JWS. jose's SignJWT with {alg:"HS256"} writes exactly -// {"alg":"HS256"} as the protected header: no typ, no kid. -func (s signer) sign(claims tokenClaims) (string, error) { - payload, err := json.Marshal(claims) - if err != nil { - return "", err - } - header := []byte(`{"alg":"HS256"}`) - encoding := base64.RawURLEncoding - signing := make([]byte, 0, - encoding.EncodedLen(len(header))+1+encoding.EncodedLen(len(payload))) - signing = encoding.AppendEncode(signing, header) - signing = append(signing, '.') - signing = encoding.AppendEncode(signing, payload) - mac := hmac.New(sha256.New, s.apiSecret) - mac.Write(signing) - token := append(signing, '.') - token = encoding.AppendEncode(token, mac.Sum(nil)) - return string(token), nil -} - -// serviceToken reproduces ServiceBase.authHeader: a 10 minute admin token -// carrying one video grant and no subject. -func (s signer) serviceToken(grant videoGrant) (string, error) { - notBefore := time.Now().Unix() - return s.sign(tokenClaims{ - Iss: s.apiKey, - Nbf: notBefore, - Exp: notBefore + serviceTokenTTLSeconds, - Video: grant, - }) -} - -// TokenRequest is SfuTokenRequest. -type TokenRequest struct { - RoomID string - Role protocol.Role - PeerID string - ShareGeneration string - PublicationGeneration string -} - -// TokenIssuer is SfuTokenIssuer. Issuing is local HMAC work, so unlike -// RoomControl it takes no context. -type TokenIssuer interface { - IssueToken(request TokenRequest) (string, error) -} - -// LiveKitTokenIssuerOptions is LiveKitTokenIssuerOptions. -type LiveKitTokenIssuerOptions struct { - APIKey string - APISecret string - MaxViewersPerRoom int -} - -// LiveKitTokenIssuer is LiveKitTokenIssuer. -type LiveKitTokenIssuer struct { - signer signer -} - -var _ TokenIssuer = (*LiveKitTokenIssuer)(nil) - -// NewLiveKitTokenIssuer mirrors the LiveKitTokenIssuer constructor. -func NewLiveKitTokenIssuer(options LiveKitTokenIssuerOptions) (*LiveKitTokenIssuer, error) { - if options.APIKey == "" || len(options.APISecret) < minAPISecretBytes { - return nil, errors.New("LiveKit API credentials are invalid") - } - if err := validateViewerLimit(options.MaxViewersPerRoom); err != nil { - return nil, err - } - return &LiveKitTokenIssuer{signer: newSigner(options.APIKey, options.APISecret)}, nil -} - -// IssueToken ports issueToken. -func (issuer *LiveKitTokenIssuer) IssueToken(request TokenRequest) (string, error) { - if !roomIDPattern.MatchString(request.RoomID) { - return "", errors.New("LiveKit room ID is invalid") - } - if !protocol.ValidOpaqueID(request.PeerID) || - !protocol.ValidOpaqueID(request.ShareGeneration) || - !protocol.ValidOpaqueID(request.PublicationGeneration) { - return "", errors.New("LiveKit participant identity is invalid") - } - - isHost := request.Role == protocol.RoleHost - // The validation above is a superset of managedRoomName's, so this cannot - // fail; the TypeScript relied on the same ordering. - room := ManagedRoomName(ResourceFence{ - RoomID: request.RoomID, - ShareGeneration: request.ShareGeneration, - PublicationGeneration: request.PublicationGeneration, - }) - identity := "host" - if !isHost { - identity = "viewer:" + request.PeerID - } - grant := videoGrant{ - RoomJoin: true, - Room: room, - CanPublish: boolPointer(isHost), - CanSubscribe: boolPointer(!isHost), - CanPublishData: boolPointer(false), - CanUpdateOwnMetadata: boolPointer(false), - } - if isHost { - grant.CanPublishSources = []string{ - trackSourceScreenShare, - trackSourceScreenShareAudio, - } - } - // One clock read: jose called new Date() once per setter, so across a - // second boundary its exp - nbf could be 299. Here it is always 300. - notBefore := time.Now().Unix() - return issuer.signer.sign(tokenClaims{ - Iss: issuer.signer.apiKey, - Sub: identity, - Nbf: notBefore, - Exp: notBefore + tokenTTLSeconds, - Video: grant, - }) -} - -// validateViewerLimit is the shared viewer-limit guard of both LiveKit -// constructors. Number.isSafeInteger is implicit in Go's int. -func validateViewerLimit(maxViewersPerRoom int) error { - if maxViewersPerRoom < 1 || maxViewersPerRoom > protocol.MaxViewersPerRoomLimit { - return errors.New("LiveKit viewer limit is invalid") - } - return nil -} - -func boolPointer(value bool) *bool { return &value } diff --git a/internal/server/sfu/token_test.go b/internal/server/sfu/token_test.go deleted file mode 100644 index 7d92494..0000000 --- a/internal/server/sfu/token_test.go +++ /dev/null @@ -1,225 +0,0 @@ -package sfu - -// Ported from tests/livekit-token.test.ts. The TypeScript verified with the -// SDK's TokenVerifier; here the JWT is split, the HS256 signature recomputed -// and the payload compared to the claim set recorded in evidence/livekit.md -// section 2. - -import ( - "crypto/hmac" - "crypto/sha256" - "encoding/base64" - "encoding/json" - "reflect" - "strings" - "testing" - "time" -) - -const ( - testAPIKey = "test-api-key" - // Placeholder credential copied from tests/livekit-token.test.ts. - testAPISecret = "ssssssssssssssssssssssssssssssss" - testShareGeneration = "share_generation_12345678" - testPublicationGeneration = "publication_12345678" -) - -// decodeToken checks the signature and returns the header and payload claims. -func decodeToken(t *testing.T, token string) (map[string]any, map[string]any) { - t.Helper() - segments := strings.Split(token, ".") - if len(segments) != 3 { - t.Fatalf("token has %d segments, want 3", len(segments)) - } - mac := hmac.New(sha256.New, []byte(testAPISecret)) - mac.Write([]byte(segments[0] + "." + segments[1])) - want := base64.RawURLEncoding.EncodeToString(mac.Sum(nil)) - if segments[2] != want { - t.Fatal("HS256 signature does not verify with the API secret") - } - return decodeSegment(t, segments[0]), decodeSegment(t, segments[1]) -} - -func decodeSegment(t *testing.T, segment string) map[string]any { - t.Helper() - raw, err := base64.RawURLEncoding.DecodeString(segment) - if err != nil { - t.Fatalf("base64: %v", err) - } - var claims map[string]any - if err := json.Unmarshal(raw, &claims); err != nil { - t.Fatalf("json: %v", err) - } - return claims -} - -func testIssuer(t *testing.T, maxViewersPerRoom int) *LiveKitTokenIssuer { - t.Helper() - issuer, err := NewLiveKitTokenIssuer(LiveKitTokenIssuerOptions{ - APIKey: testAPIKey, - APISecret: testAPISecret, - MaxViewersPerRoom: maxViewersPerRoom, - }) - if err != nil { - t.Fatalf("NewLiveKitTokenIssuer: %v", err) - } - return issuer -} - -func issue(t *testing.T, issuer *LiveKitTokenIssuer, request TokenRequest) (map[string]any, map[string]any) { - t.Helper() - token, err := issuer.IssueToken(request) - if err != nil { - t.Fatalf("IssueToken: %v", err) - } - return decodeToken(t, token) -} - -func TestTokenIssuesGenerationBoundHostToken(t *testing.T) { - before := time.Now().Unix() - header, claims := issue(t, testIssuer(t, 8), TokenRequest{ - RoomID: "42", - Role: "host", - PeerID: "host_peer_12345678", - ShareGeneration: testShareGeneration, - PublicationGeneration: testPublicationGeneration, - }) - after := time.Now().Unix() - - // jose emitted exactly {"alg":"HS256"}: no typ, no kid. - if !reflect.DeepEqual(header, map[string]any{"alg": "HS256"}) { - t.Fatalf("header = %v", header) - } - if claims["iss"] != testAPIKey || claims["sub"] != "host" { - t.Fatalf("iss/sub = %v/%v", claims["iss"], claims["sub"]) - } - notBefore, expires := claims["nbf"].(float64), claims["exp"].(float64) - if expires-notBefore != 5*60 { - t.Fatalf("exp - nbf = %v, want 300", expires-notBefore) - } - if notBefore < float64(before) || notBefore > float64(after) { - t.Fatalf("nbf = %v, want it inside [%d,%d]", notBefore, before, after) - } - if _, present := claims["iat"]; present { - t.Fatal("iat must be absent") - } - if _, present := claims["identity"]; present { - t.Fatal("identity must be absent") - } - if _, present := claims["roomConfig"]; present { - t.Fatal("roomConfig must be absent") - } - want := map[string]any{ - "roomJoin": true, - "room": "screener-v1.42." + testShareGeneration + "." + testPublicationGeneration, - "canPublish": true, - "canSubscribe": false, - "canPublishData": false, - "canUpdateOwnMetadata": false, - "canPublishSources": []any{"screen_share", "screen_share_audio"}, - } - assertVideoGrant(t, claims, want) -} - -func TestTokenIssuesSubscribeOnlyViewerToken(t *testing.T) { - const viewerPeerID = "viewer_root_12345678" - _, claims := issue(t, testIssuer(t, 3), TokenRequest{ - RoomID: "7", - Role: "viewer", - PeerID: viewerPeerID, - ShareGeneration: testShareGeneration, - PublicationGeneration: testPublicationGeneration, - }) - - if claims["sub"] != "viewer:"+viewerPeerID { - t.Fatalf("sub = %v", claims["sub"]) - } - // canPublishSources is dropped entirely for a viewer, and canPublish must - // be present as false: an absent flag reads as granted to LiveKit. - assertVideoGrant(t, claims, map[string]any{ - "roomJoin": true, - "room": "screener-v1.7." + testShareGeneration + "." + testPublicationGeneration, - "canPublish": false, - "canSubscribe": true, - "canPublishData": false, - "canUpdateOwnMetadata": false, - }) -} - -func TestTokenIsolatesPublicationGenerations(t *testing.T) { - issuer := testIssuer(t, 8) - request := TokenRequest{ - RoomID: "7", - Role: "host", - PeerID: "host_peer_12345678", - ShareGeneration: testShareGeneration, - } - request.PublicationGeneration = "generation_first_12345678" - _, first := issue(t, issuer, request) - request.PublicationGeneration = "generation_second_12345678" - _, second := issue(t, issuer, request) - - firstRoom := first["video"].(map[string]any)["room"] - secondRoom := second["video"].(map[string]any)["room"] - if firstRoom == secondRoom { - t.Fatalf("both generations share the room %v", firstRoom) - } -} - -func TestTokenRejectsInvalidRequestsAndCredentials(t *testing.T) { - issuer := testIssuer(t, 8) - base := TokenRequest{ - RoomID: "42", - Role: "host", - PeerID: "host_peer_12345678", - ShareGeneration: testShareGeneration, - PublicationGeneration: testPublicationGeneration, - } - for name, mutate := range map[string]func(*TokenRequest){ - "zero room": func(request *TokenRequest) { request.RoomID = "0" }, - "leading zero": func(request *TokenRequest) { request.RoomID = "042" }, - "thirteen digits": func(request *TokenRequest) { - request.RoomID = "1234567890123" - }, - "empty room": func(request *TokenRequest) { request.RoomID = "" }, - "short peer": func(request *TokenRequest) { request.PeerID = "short" }, - "dotted share": func(request *TokenRequest) { request.ShareGeneration = "has.dot.12345678" }, - "empty publication": func(request *TokenRequest) { - request.PublicationGeneration = "" - }, - "unicode peer": func(request *TokenRequest) { request.PeerID = "peer_é_12345678" }, - } { - request := base - mutate(&request) - if _, err := issuer.IssueToken(request); err == nil { - t.Fatalf("%s: want a rejection", name) - } - } - - for name, options := range map[string]LiveKitTokenIssuerOptions{ - "empty key": {APIKey: "", APISecret: testAPISecret, MaxViewersPerRoom: 8}, - "short secret": {APIKey: testAPIKey, APISecret: strings.Repeat("s", 31), MaxViewersPerRoom: 8}, - "no viewers": {APIKey: testAPIKey, APISecret: testAPISecret, MaxViewersPerRoom: 0}, - "too many viewers": { - APIKey: testAPIKey, APISecret: testAPISecret, MaxViewersPerRoom: 21, - }, - } { - if _, err := NewLiveKitTokenIssuer(options); err == nil { - t.Fatalf("%s: want a rejection", name) - } - } -} - -// assertVideoGrant compares the whole claim: an extra or missing key is a -// different grant, which is the point of canPublishSources being absent for a -// viewer and canPublish being present as false. -func assertVideoGrant(t *testing.T, claims map[string]any, want map[string]any) { - t.Helper() - grant, ok := claims["video"].(map[string]any) - if !ok { - t.Fatalf("video = %v", claims["video"]) - } - if !reflect.DeepEqual(grant, want) { - t.Fatalf("video = %v, want %v", grant, want) - } -} diff --git a/internal/server/signal/router.go b/internal/server/signal/router.go index 487c1cb..209bf7b 100644 --- a/internal/server/signal/router.go +++ b/internal/server/signal/router.go @@ -4,8 +4,8 @@ package signal // // The router owns no lock. signal.Server.mu (routerOptions.mu) is held by the // caller of every method except close, hooks are called with it held, and the -// goroutines the router starts (pump driver, fresh SFU config, host-offline -// probe, drains, timer callbacks) take it themselves. Every TS `await` is an +// goroutines the router starts (pump driver, media prepares, drains and timer +// callbacks) take it themselves. Every I/O window is an // unlock / I/O / lock window followed by exactly the guard the TS code ran // after that await (map C:/tmp/screener-go/maps/effect-layer.md, section 2.2). @@ -16,7 +16,6 @@ import ( "errors" "fmt" "log/slog" - "net/url" "os" "strings" "sync" @@ -31,14 +30,12 @@ import ( const ( defaultRouteOperationTimeoutMs int64 = 20_000 - defaultSfuDrainRetryMs int64 = 1_000 - defaultHostOfflineCheckMs int64 = 5_000 ) -// routeDebugEnabled ports debuglog("screener-route"): SCREENER_DEBUG=route -// (D8), read once like NODE_DEBUG. The route package keeps the same flag -// unexported, so this is its second reader. -var routeDebugEnabled = routeDebugFlag(os.Getenv("SCREENER_DEBUG")) +func routeDebugEnabled() bool { + return routeDebugFlag(os.Getenv("SCREENER_DEBUG")) || + slog.Default().Enabled(context.Background(), slog.LevelDebug) +} func routeDebugFlag(value string) bool { for _, section := range strings.Split(value, ",") { @@ -51,13 +48,9 @@ func routeDebugFlag(value string) bool { // sfuFallback is SfuFallbackOptions. A zero timeout means the TS default. type sfuFallback struct { - url string - tokenIssuer sfu.TokenIssuer - admission *sfu.Admission - roomControl sfu.RoomControl - prepareTimeoutMs int64 - drainRetryMs int64 - hostOfflineCheckMs int64 + media sfu.Runtime + admission *sfu.Admission + prepareTimeoutMs int64 } // routerHooks are the signaling server callbacks of HybridMediaRouterOptions. @@ -153,17 +146,6 @@ type roomRuntime struct { pumping bool deadlineStop func() bool deadlineGeneration uint64 - sfuRefreshesInFlight map[string]struct{} -} - -// hostOfflineCheck is HostOfflineCheck; the timer callback compares the -// registered pointer with the one it was armed with. -type hostOfflineCheck struct { - hostPeerID string - fence sfu.ResourceFence - hostSessionID string - connectionID string - stop func() bool } // router is HybridMediaRouter. @@ -178,19 +160,14 @@ type router struct { // O14, O16, O29: rooms, drain tasks and waiters are walked in insertion // order. - rooms ordered.Map[string, *roomRuntime] - resourceWaiters ordered.Map[string, struct{}] - sfuDrainTasks ordered.Map[string, *sfuDrainTask] - hostOfflineChecks ordered.Map[string, *hostOfflineCheck] - // roomTurns is the per managed-room-name FIFO of takeTurn. - roomTurns map[string]chan struct{} + rooms ordered.Map[string, *roomRuntime] + resourceWaiters ordered.Map[string, struct{}] + sfuDrainTasks ordered.Map[sfu.SubscriptionFence, *sfuDrainTask] // inflight counts the goroutines that will re-acquire mu: the pump // drivers, fresh SFU config issues and drains. It is the TS microtask // backlog; tests wait for it to reach zero where the TS awaited. inflight int closing bool - // closeCtx bounds every drain started once close began (D6). - closeCtx context.Context } // newRouter is the HybridMediaRouter constructor. @@ -198,12 +175,7 @@ func newRouter(options routerOptions) *router { if err := protocol.AssertEndpointMediaCopyCapacity(options.endpointMediaCopyCapacity); err != nil { panic(err) } - if options.sfu != nil { - // TS: new URL(options.sfuFallback.url) throws on a relative URL. - if parsed, err := url.Parse(options.sfu.url); err != nil || parsed.Scheme == "" { - panic(errors.New("SFU fallback URL is invalid")) - } - } + r := &router{ mu: options.mu, store: options.store, @@ -212,7 +184,6 @@ func newRouter(options routerOptions) *router { hooks: options.hooks, now: options.now, afterFunc: options.afterFunc, - roomTurns: map[string]chan struct{}{}, } if r.now == nil { r.now = func() int64 { return time.Now().UnixMilli() } @@ -237,23 +208,13 @@ func (r *router) io(fn func()) { fn() } -// close ports close(). mu must be held, as for every other router method: the -// synchronous prefix (closing flag, timers, drains scheduled) runs in the -// caller's critical section exactly as the TS ran it before its first await, -// and mu is released only inside awaitDrain, around each LiveKit wait (R1, -// R2). Drains run sequentially in insertion order (O16) and their errors are -// joined; ctx bounds every wait and every close-initiated LiveKit call. +// close retires room operations and waits for their exact physical resources. func (r *router) close(ctx context.Context) error { r.closing = true - r.closeCtx = ctx for _, roomID := range r.rooms.Keys() { r.clearRoom(roomID) } r.resourceWaiters.Clear() - for _, check := range r.hostOfflineChecks.Values() { - check.stop() - } - r.hostOfflineChecks.Clear() fallback := r.sfu if fallback == nil { return nil @@ -262,7 +223,7 @@ func (r *router) close(ctx context.Context) error { r.scheduleSfuPublicationDrain(fence) } type entry struct { - key string + key sfu.SubscriptionFence task *sfuDrainTask } var snapshot []entry @@ -272,7 +233,6 @@ func (r *router) close(ctx context.Context) error { var errs []error for _, current := range snapshot { key, task := current.key, current.task - r.clearDrainRetry(task) if task.operation != nil { if err := r.awaitDrain(ctx, task.operation); err != nil { errs = append(errs, err) @@ -280,13 +240,13 @@ func (r *router) close(ctx context.Context) error { } } if registered, _ := r.sfuDrainTasks.Get(key); registered == task { - if err := r.awaitDrain(ctx, r.runSfuDrain(ctx, key, task)); err != nil { + if err := r.awaitDrain(ctx, r.runSfuDrain(key, task)); err != nil { errs = append(errs, err) } } } if len(errs) > 0 { - return fmt.Errorf("LiveKit drain failed during shutdown: %w", errors.Join(errs...)) + return fmt.Errorf("SFU drain failed during shutdown: %w", errors.Join(errs...)) } return nil } @@ -304,7 +264,6 @@ func (r *router) connectParticipant(input authenticatedRouteParticipant) hybridA r.releaseResources(rm.controller.RebindHostIdentity(input.peerID, input.sessionID, r.nowPtr())) } rm.hostPeerID = input.peerID - r.cancelHostOfflineCheck(input.roomID) if rm.controller == nil { r.createController(input.roomID, rm, input) } @@ -341,7 +300,7 @@ func (r *router) completeAuthentication(participant authenticatedRouteParticipan } r.broadcastActive(participant.roomID, rm) // TS: void this.sendFreshSfuConfig(participant).catch(() => undefined) - r.sendFreshSfuConfig(participant, nil, nil) + r.sendFreshSfuConfig(participant) r.requestPump(participant.roomID) } @@ -376,7 +335,7 @@ func (r *router) observeQualityEvidence(roomID string, evidence route.RouteQuali if rm, _ := r.rooms.Get(roomID); rm != nil && rm.controller != nil { result = rm.controller.ObserveQualityEvidence(evidence) } - if result != route.QualityEvidenceRejected && routeDebugEnabled { + if result != route.QualityEvidenceRejected && routeDebugEnabled() { upstream := "sfu" if evidence.Upstream.Kind == route.UpstreamPeer { upstream = "p2p:" + r.debugPeer(roomID, evidence.Upstream.PeerID) @@ -445,7 +404,7 @@ func (r *router) observeSenderQualityEvidence(participant authenticatedRoutePart AcceptedAtMs: r.now(), } result := controller.ObserveSenderQualityEvidence(input, r.commitReservation) - if result.Accepted && routeDebugEnabled { + if result.Accepted && routeDebugEnabled() { diagnostics := message.Diagnostics details := []any{ "parent", r.debugPeer(participant.roomID, participant.peerID), @@ -515,7 +474,7 @@ func (r *router) observeSfuPublisherQualityEvidence(participant authenticatedRou AcceptedAtMs: r.now(), } result := controller.ObserveSfuPublisherQualityEvidence(input, r.commitReservation) - if result.Accepted && routeDebugEnabled { + if result.Accepted && routeDebugEnabled() { diagnostics := message.Diagnostics details := []any{"routeRevision", int64(message.RouteRevision)} details = append(details, copyContext.kv()...) @@ -691,7 +650,7 @@ func (r *router) peerSignalAuthorization(input peerSignalInput) signalAuthorizat // debugPeerSignal ports debugPeerSignal. func (r *router) debugPeerSignal(input peerSignalDebugInput) { - if !routeDebugEnabled { + if !routeDebugEnabled() { return } var authorization any @@ -956,53 +915,11 @@ func (r *router) handleRouteFailed(participant authenticatedRouteParticipant, me r.requestPump(participant.roomID) } -// refreshSfu ports refreshSfu. func (r *router) refreshSfu(participant authenticatedRouteParticipant, revision int64) { rm, _ := r.rooms.Get(participant.roomID) - if rm == nil || rm.controller == nil { - return + if rm != nil && rm.controller != nil && revision <= rm.controller.Revision() { + r.sendFreshSfuConfig(participant) } - currentRevision := rm.controller.Revision() - if revision > currentRevision { - return - } - key := fmt.Sprintf("%d\x00%s\x00%s", currentRevision, participant.peerID, participant.sessionID) - if _, inFlight := rm.sfuRefreshesInFlight[key]; inFlight { - return - } - rm.sfuRefreshesInFlight[key] = struct{}{} - r.sendFreshSfuConfig(participant, - func() { r.failCurrentSfuRoute(participant, currentRevision) }, - func() { delete(rm.sfuRefreshesInFlight, key) }) -} - -// failCurrentSfuRoute ports failCurrentSfuRoute. -func (r *router) failCurrentSfuRoute(participant authenticatedRouteParticipant, revision int64) { - rm, _ := r.rooms.Get(participant.roomID) - if rm == nil || rm.controller == nil { - return - } - snapshot := rm.controller.Snapshot() - if snapshot.Revision != revision { - return - } - connectionID := "" - if participant.peerID == rm.hostPeerID { - if snapshot.HostPublication != nil { - connectionID = snapshot.HostPublication.ConnectionID - } - } else if edge, ok := snapshot.UpstreamByViewer.Get(participant.peerID); ok && edge.Kind == route.UpstreamSfu { - connectionID = edge.ConnectionID - } - if connectionID == "" { - return - } - r.handleRouteFailed(participant, protocol.RouteFailedMessage{ - Type: "route-failed", - Revision: protocol.Int(revision), - Phase: "active", - ConnectionID: &connectionID, - }) } // disconnectParticipant ports disconnectParticipant. @@ -1011,9 +928,6 @@ func (r *router) disconnectParticipant(roomID, peerID, sessionID string) { if rm == nil || rm.controller == nil || !rm.controller.DisconnectSession(peerID, sessionID) { return } - if peerID == rm.hostPeerID { - r.scheduleHostOfflineCheck(roomID, peerID) - } r.requestPump(roomID) } @@ -1059,7 +973,7 @@ func (r *router) deleteRoom(roomID string) { func (r *router) room(roomID string) *roomRuntime { rm, ok := r.rooms.Get(roomID) if !ok { - rm = &roomRuntime{sfuRefreshesInFlight: map[string]struct{}{}} + rm = &roomRuntime{} r.rooms.Set(roomID, rm) } return rm @@ -1106,7 +1020,6 @@ func (r *router) clearRoom(roomID string) { return } r.clearDeadline(rm) - r.cancelHostOfflineCheck(roomID) controller := rm.controller rm.controller = nil rm.requested = false @@ -1261,7 +1174,7 @@ func (r *router) broadcastActive(roomID string, rm *roomRuntime) { return } snapshot := rm.controller.Snapshot() - if routeDebugEnabled { + if routeDebugEnabled() { type routeEntry struct { Child string `json:"child"` Parent string `json:"parent"` @@ -1505,7 +1418,7 @@ func (r *router) debugTuple(roomID string, tuple route.CandidateTuple) string { // debug ports debug(): one sanitised slog event. details are key/value // pairs; never pass raw peer, session or connection IDs (use debugPeer). func (r *router) debug(roomID, event string, details ...any) { - if !routeDebugEnabled { + if !routeDebugEnabled() { return } args := make([]any, 0, 4+len(details)) diff --git a/internal/server/signal/router_pump.go b/internal/server/signal/router_pump.go index 618042e..baef594 100644 --- a/internal/server/signal/router_pump.go +++ b/internal/server/signal/router_pump.go @@ -1,13 +1,8 @@ package signal -// The route pump of hybrid-media-router.ts (requestPump 1114, pumpRoom 1135, -// prepareCandidate 1303, prepareNewPublication 1409, sendPrepareMessages 1492, -// scheduleDeadline 1822), following map section 7 step by step. The only -// unlock windows are inside prepareCandidate (R5, R6, R8, R9, R10); the -// cursor guard captured before them is the sole staleness detector. +// Room authority prepares one candidate at a time; media signaling follows admission. import ( - "context" "log/slog" "time" @@ -28,7 +23,7 @@ type preparedCandidate struct { viewerSfuConfig *protocol.SfuConfigMessage } -// prepareResult is PrepareResult. +// prepareResult is one admitted candidate or a typed rejection. type prepareResult struct { denied bool rejectionBucket route.RejectionBucket @@ -39,18 +34,13 @@ func denied(bucket route.RejectionBucket) prepareResult { return prepareResult{denied: true, rejectionBucket: bucket} } -// pumpCandidate is the pump loop state carried across the yield at -// `await prepareCandidate`: the selected slot with its cursor guard and the -// outcome of prepareCandidate's synchronous prefix. io is the awaited -// remainder (drain wait, token issue, LiveKit room creation), nil when -// result is already settled. +// pumpCandidate keeps the admitted result across the pump driver's queued turn. type pumpCandidate struct { operation *route.OperationSnapshot plan route.CandidatePlan guard route.CandidateCursorGuard revision int64 result prepareResult - io func() prepareResult } // requestPump ports requestPump: the single-flight driver. Every caller @@ -59,7 +49,7 @@ type pumpCandidate struct { // The TS driver was an async IIFE, so it ran synchronously inside the // caller up to its first await: the first pumpRoom iteration through the // synchronous prefix of prepareCandidate (reconcile, candidate selection, -// the admission reservations and the LiveKit createRoom initiation). That +// the admission reservations). That // prefix is observable: the calls that follow requestPump in the same // handler see its reconcile, and two rooms woken by one drain reserve // SFU capacity in wake order (map O29). It therefore runs here in the @@ -137,11 +127,7 @@ func (r *router) drivePump(roomID string, rm *roomRuntime, controller *route.Con } } -// pumpRoom ports pumpRoom. mu is held on entry and exit; the candidate's io -// releases it around I/O. resume continues an iteration suspended at -// `await prepareCandidate`; with yieldAtPrepare the first candidate is -// returned after prepareCandidate's synchronous prefix instead of being -// awaited (the TS suspension point). Otherwise nil. +// pumpRoom keeps admission ordered while allowing the driver to yield before signaling. func (r *router) pumpRoom( roomID string, rm *roomRuntime, @@ -168,7 +154,7 @@ func (r *router) pumpRoom( return nil } candidate = selected - candidate.result, candidate.io = r.prepareCandidate( + candidate.result = r.prepareCandidate( roomID, rm, controller, candidate.operation, candidate.plan, candidate.revision) if yieldAtPrepare { return &candidate @@ -176,9 +162,6 @@ func (r *router) pumpRoom( } operation, plan, guard := candidate.operation, candidate.plan, candidate.guard preparation := candidate.result - if candidate.io != nil { - preparation = candidate.io() - } if preparation.denied { r.debug(roomID, "candidate-preparation-denied", "child", r.debugPeer(roomID, operation.ChildPeerID), @@ -310,16 +293,7 @@ func (r *router) selectCandidate( return pumpCandidate{operation: operation, plan: plan, guard: guard, revision: revision}, false } -// prepareCandidate ports prepareCandidate. It runs the synchronous prefix -// (everything the TS executed before its first await: the stale checks, -// the borrowed edge lookup, the admission reservations, the LiveKit -// createRoom initiation) with mu held and returns either a settled result -// or the awaited remainder as io. io is called with mu held and releases -// it around reserveSfuSubscription's drain wait (R5/R10), the token issue -// (R6) and, for a new publication, the LiveKit room creation (R8/R9). No -// identity is re-read afterwards: the cursor guard in pumpRoom catches -// staleness. controller is the pump's captured controller (TS read -// room.controller!, the same object on every reachable path). +// prepareCandidate reserves the exact connection before any client can signal it. func (r *router) prepareCandidate( roomID string, rm *roomRuntime, @@ -327,12 +301,12 @@ func (r *router) prepareCandidate( operation *route.OperationSnapshot, plan route.CandidatePlan, revision int64, -) (prepareResult, func() prepareResult) { +) prepareResult { snapshot := controller.Snapshot() child, hasChild := r.store.GetConnectedViewer(roomID, operation.ChildPeerID) shareGeneration := r.hooks.shareGeneration(roomID) if !hasChild || child.SessionID != operation.ChildSessionID || shareGeneration == "" { - return denied(route.RejectionStale), nil + return denied(route.RejectionStale) } connectionID := opaqueID() var overlap *route.Resource @@ -347,111 +321,46 @@ func (r *router) prepareCandidate( return prepareResult{prepared: preparedCandidate{ connectionID: connectionID, reservation: route.CandidateReservation{Kind: route.ReservationDirect, Overlap: overlap}, - }}, nil + }} } fallback := r.sfu host, hasHost := r.store.GetConnectedHost(roomID) if fallback == nil || !hasHost || host.PeerID != rm.hostPeerID { - return denied(route.RejectionStale), nil + return denied(route.RejectionStale) } if plan.Tuple.Publication == route.PublicationReuse { publication := snapshot.HostPublication if publication == nil || publication.Resource == nil || publication.Resource.Kind != route.ResourceSfuPublication { - return denied(route.RejectionStale), nil + return denied(route.RejectionStale) } fence := sfu.SubscriptionFence{ ResourceFence: resourceFence(publication.Resource), ViewerPeerID: operation.ChildPeerID, + ConnectionID: connectionID, } - var borrowedEdge *route.Resource - if currentEdge, ok := snapshot.UpstreamByViewer.Get(operation.ChildPeerID); ok && - currentEdge.Kind == route.UpstreamSfu && - currentEdge.PublicationGeneration == publication.Generation && - currentEdge.Resource != nil && - currentEdge.Resource.Kind == route.ResourceSfuSubscription && - !currentEdge.Resource.Released { - borrowedEdge = currentEdge.Resource - } - // issue is the remainder after `await this.reserveSfuSubscription`. issue := func(edge *route.Resource) prepareResult { - request := sfu.TokenRequest{ - RoomID: roomID, - Role: protocol.RoleViewer, - PeerID: operation.ChildPeerID, - ShareGeneration: shareGeneration, - PublicationGeneration: publication.Generation, - } - var token string - var err error - r.io(func() { token, err = fallback.tokenIssuer.IssueToken(request) }) // R6 - if err != nil { - if borrowedEdge == nil { - r.releaseResource(edge) - } - return denied(route.RejectionCandidateFailed) - } return prepareResult{prepared: preparedCandidate{ - connectionID: connectionID, - hostSessionID: host.SessionID, - reservation: route.CandidateReservation{ - Kind: route.ReservationSfuReuse, - Edge: edge, - Borrowed: borrowedEdge != nil, - Overlap: overlap, - }, + connectionID: connectionID, hostSessionID: host.SessionID, + reservation: route.CandidateReservation{Kind: route.ReservationSfuReuse, Edge: edge, Overlap: overlap}, publicationGeneration: publication.Generation, viewerSfuConfig: &protocol.SfuConfigMessage{ - Type: "sfu-config", - Revision: protocol.Int(revision), - URL: fallback.url, - Token: token, + Type: "sfu-config", Revision: protocol.Int(revision), + PublicationGeneration: publication.Generation, ConnectionID: connectionID, }, }} } - if borrowedEdge != nil { - return prepareResult{}, func() prepareResult { return issue(borrowedEdge) } + if !fallback.admission.ReserveSubscription(fence) { + return denied(route.RejectionSfuAdmission) } - // reserveSfuSubscription (1788): a pending drain of the same - // subscription is awaited first (R10); otherwise the reservation is - // part of the synchronous prefix. - key := sfuDrainKey(drainSubscription, fence) - if pending, _ := r.sfuDrainTasks.Get(key); pending != nil && pending.operation != nil { - drain := pending.operation - return prepareResult{}, func() prepareResult { - r.io(func() { <-drain.done }) - if drain.err != nil { - return denied(route.RejectionSfuAdmission) - } - // Hazard 8: a drain that settled but left its task - // registered means refuse. - if registered, _ := r.sfuDrainTasks.Get(key); registered == pending { - return denied(route.RejectionSfuAdmission) - } - if !r.reserveSfuSubscription(fence) { - return denied(route.RejectionSfuAdmission) - } - return issue(subscriptionResource(fence)) - } - } - if !r.reserveSfuSubscription(fence) { - return denied(route.RejectionSfuAdmission), nil - } - edge := subscriptionResource(fence) - return prepareResult{}, func() prepareResult { return issue(edge) } + return issue(subscriptionResource(fence)) } return r.prepareNewPublication(roomID, operation, revision, connectionID, shareGeneration, host, overlap) } -// prepareNewPublication ports prepareNewPublication. The reservations and -// the room's FIFO turn (the TS createRoom call, registered synchronously -// by LiveKitSfuRoomControl.serialize) belong to the synchronous prefix; -// io creates the LiveKit room inside that turn and issues the two tokens -// sequentially where the TS used Promise.all (hazard 6: latency only). -// Failure releases the subscription before the publication (O26), the -// reverse of releaseResources. +// One candidate reserves both publication and first subscription atomically. func (r *router) prepareNewPublication( roomID string, operation *route.OperationSnapshot, @@ -460,7 +369,7 @@ func (r *router) prepareNewPublication( shareGeneration string, host room.ConnectedPeer, overlap *route.Resource, -) (prepareResult, func() prepareResult) { +) prepareResult { fallback := r.sfu publicationGeneration := opaqueID() publicationFence := sfu.ResourceFence{ @@ -471,76 +380,34 @@ func (r *router) prepareNewPublication( subscriptionFence := sfu.SubscriptionFence{ ResourceFence: publicationFence, ViewerPeerID: operation.ChildPeerID, + ConnectionID: connectionID, } if !fallback.admission.ReservePublication(publicationFence) { - return denied(route.RejectionSfuAdmission), nil + return denied(route.RejectionSfuAdmission) } if !fallback.admission.ReserveSubscription(subscriptionFence) { fallback.admission.BeginDrain(publicationFence) r.scheduleSfuPublicationDrain(publicationFence) - return denied(route.RejectionSfuAdmission), nil + return denied(route.RejectionSfuAdmission) } subscription := subscriptionResource(subscriptionFence) publicationConnectionID := publicationGeneration publication := publicationResource(publicationFence) - turn := r.takeTurn(sfu.ManagedRoomName(publicationFence)) - return prepareResult{}, func() prepareResult { - var err error - var hostToken, viewerToken string - r.io(func() { - turn.run(func() { - err = fallback.roomControl.CreateRoom(context.Background(), publicationFence) // R8 - }) - if err == nil { // R9 - hostToken, err = fallback.tokenIssuer.IssueToken(sfu.TokenRequest{ - RoomID: roomID, - Role: protocol.RoleHost, - PeerID: host.PeerID, - ShareGeneration: shareGeneration, - PublicationGeneration: publicationGeneration, - }) - } - if err == nil { - viewerToken, err = fallback.tokenIssuer.IssueToken(sfu.TokenRequest{ - RoomID: roomID, - Role: protocol.RoleViewer, - PeerID: operation.ChildPeerID, - ShareGeneration: shareGeneration, - PublicationGeneration: publicationGeneration, - }) - } - }) - r.dropTurn(turn) - if err != nil { - r.releaseResource(subscription) - r.releaseResource(publication) - return denied(route.RejectionCandidateFailed) - } - return prepareResult{prepared: preparedCandidate{ - connectionID: connectionID, - hostSessionID: host.SessionID, - publicationGeneration: publicationGeneration, - publicationConnectionID: publicationConnectionID, - reservation: route.CandidateReservation{ - Kind: route.ReservationSfuCreate, - Edge: subscription, - Publication: publication, - Overlap: overlap, - }, - hostSfuConfig: &protocol.SfuConfigMessage{ - Type: "sfu-config", - Revision: protocol.Int(revision), - URL: fallback.url, - Token: hostToken, - }, - viewerSfuConfig: &protocol.SfuConfigMessage{ - Type: "sfu-config", - Revision: protocol.Int(revision), - URL: fallback.url, - Token: viewerToken, - }, - }} - } + return prepareResult{prepared: preparedCandidate{ + connectionID: connectionID, hostSessionID: host.SessionID, + publicationGeneration: publicationGeneration, publicationConnectionID: publicationConnectionID, + reservation: route.CandidateReservation{ + Kind: route.ReservationSfuCreate, Edge: subscription, Publication: publication, Overlap: overlap, + }, + hostSfuConfig: &protocol.SfuConfigMessage{ + Type: "sfu-config", Revision: protocol.Int(revision), + PublicationGeneration: publicationGeneration, ConnectionID: publicationConnectionID, + }, + viewerSfuConfig: &protocol.SfuConfigMessage{ + Type: "sfu-config", Revision: protocol.Int(revision), + PublicationGeneration: publicationGeneration, ConnectionID: connectionID, + }, + }} } // sendPrepareMessages ports sendPrepareMessages (O24: fixed order). It sends @@ -587,6 +454,9 @@ func (r *router) sendPrepareMessages( r.hooks.sendToSession(child.SessionID, childUpdate) if prepared.viewerSfuConfig != nil { r.hooks.sendToSession(child.SessionID, *prepared.viewerSfuConfig) + if prepared.hostSfuConfig == nil { + r.prepareSfuSubscriber(roomID, operation, prepared.publicationGeneration) + } } } @@ -596,7 +466,7 @@ func (r *router) sendPrepareMessages( func (r *router) scheduleDeadline(roomID string, rm *roomRuntime, operation *route.OperationSnapshot) { r.clearDeadline(rm) wakeInMs := max(int64(0), operation.WakeAtMs-r.now()) - if routeDebugEnabled { + if routeDebugEnabled() { var current any if operation.Current != nil { current = r.debugTuple(roomID, operation.Current.Tuple) @@ -685,6 +555,7 @@ func subscriptionResource(fence sfu.SubscriptionFence) *route.Resource { ShareGeneration: fence.ShareGeneration, PublicationGeneration: fence.PublicationGeneration, ViewerPeerID: fence.ViewerPeerID, + ConnectionID: fence.ConnectionID, } } @@ -706,5 +577,5 @@ func resourceFence(resource *route.Resource) sfu.ResourceFence { } func subscriptionFence(resource *route.Resource) sfu.SubscriptionFence { - return sfu.SubscriptionFence{ResourceFence: resourceFence(resource), ViewerPeerID: resource.ViewerPeerID} + return sfu.SubscriptionFence{ResourceFence: resourceFence(resource), ViewerPeerID: resource.ViewerPeerID, ConnectionID: resource.ConnectionID} } diff --git a/internal/server/signal/router_sfu.go b/internal/server/signal/router_sfu.go index 06fca5c..28c698d 100644 --- a/internal/server/signal/router_sfu.go +++ b/internal/server/signal/router_sfu.go @@ -1,14 +1,8 @@ package signal -// The SFU side of hybrid-media-router.ts: reservation commit and release -// (1734-1786), subscription reservation across a pending drain (1788), -// resource waiters (1811), fresh SFU configuration (1865), the host-offline -// probe (1903-2013) and the LiveKit drains (2015-2089). - import ( "context" "errors" - "time" "github.com/TNTcraftHIM/Screener/internal/server/protocol" "github.com/TNTcraftHIM/Screener/internal/server/route" @@ -22,16 +16,11 @@ const ( drainSubscription drainKind = "subscription" ) -// sfuDrainTask is SfuDrainTask; fence.ViewerPeerID is "" for a publication. -// operation is the in-flight drain (TS task.operation); retryGeneration is -// bumped by clearDrainRetry so a retry callback that lost the Stop race sees -// it is stale (D5). +// sfuDrainTask owns one asynchronous exact physical close. type sfuDrainTask struct { - kind drainKind - fence sfu.SubscriptionFence - operation *drainOperation - retryStop func() bool - retryGeneration uint64 + kind drainKind + fence sfu.SubscriptionFence + operation *drainOperation } // drainOperation is the promise runSfuDrain tracked: done is closed after err @@ -41,51 +30,6 @@ type drainOperation struct { err error } -// turn is one slot in the per managed-room-name FIFO. -// -// The TS LiveKitSfuRoomControl.serialize registered every operation -// synchronously at call time, so operations on one managed room ran in the -// order the router decided them. In Go the RoomControl call happens after mu -// is released, and two goroutines could reach LiveKit in the opposite order -// (a drain deciding after a create but deleting the room before it exists). -// takeTurn therefore reserves the slot while mu is still held; run waits for -// the predecessor outside mu, performs the call and releases the successor; -// dropTurn forgets the tail entry once mu is held again. The sfu package's -// own serialisation then sees calls already in decision order. Only the -// mutating calls (CreateRoom, DeleteRoom, DrainSubscription) take a turn: -// their relative order is the correctness property; HostParticipantExists -// is a read whose result is re-validated against the controller anyway, and -// the TS suite's fake control never queued a drain behind it. -type turn struct { - name string - previous chan struct{} - done chan struct{} -} - -// takeTurn reserves the next slot for roomName. mu must be held. -func (r *router) takeTurn(roomName string) *turn { - done := make(chan struct{}) - current := &turn{name: roomName, previous: r.roomTurns[roomName], done: done} - r.roomTurns[roomName] = done - return current -} - -// run performs op in turn order. mu must NOT be held. -func (t *turn) run(op func()) { - if t.previous != nil { - <-t.previous - } - defer close(t.done) - op() -} - -// dropTurn removes the FIFO tail once it is this turn. mu must be held. -func (r *router) dropTurn(t *turn) { - if r.roomTurns[t.name] == t.done { - delete(r.roomTurns, t.name) - } -} - // commitReservation ports commitReservation (O30: drained generations in // admission order). func (r *router) commitReservation(reservation route.CandidateReservation) bool { @@ -118,8 +62,7 @@ func (r *router) releaseReservation(reservation route.CandidateReservation) { r.releaseResources(reservationResources(reservation)) } -// releaseResources ports releaseResources (O25). Publication drains own -// accounting; subscription drains only remove exact participants. +// releaseResources retires publications before their exact subscriptions. func (r *router) releaseResources(resources []*route.Resource) { for _, resource := range resources { if resource.Kind == route.ResourceSfuPublication { @@ -162,22 +105,6 @@ func (r *router) releaseSubscription(fence sfu.SubscriptionFence) bool { return accepted } -// reserveSfuSubscription ports the synchronous tail of reserveSfuSubscription -// (1803-1808); the pending-drain wait before it (R10) lives in -// prepareCandidate, where the TS suspended. -func (r *router) reserveSfuSubscription(fence sfu.SubscriptionFence) bool { - fallback := r.sfu - if fallback == nil { - return false - } - key := sfuDrainKey(drainSubscription, fence) - if task, _ := r.sfuDrainTasks.Get(key); task != nil && task.kind == drainSubscription { - r.clearDrainRetry(task) - r.sfuDrainTasks.Delete(key) - } - return fallback.admission.ReserveSubscription(fence) -} - // wakeResourceWaiters ports wakeResourceWaiters (O29: snapshot order). func (r *router) wakeResourceWaiters() { roomIDs := r.resourceWaiters.Keys() @@ -192,225 +119,45 @@ func (r *router) wakeResourceWaiters() { } } -// sendFreshSfuConfig ports sendFreshSfuConfig (R11). The pre-await guards run -// in the caller's critical section, as the TS ran them before its first -// await; the token is issued in a goroutine with mu released and the -// post-await guards re-run under mu with the pre-await snapshot. failed runs -// where the TS `.catch` ran and settled where `.finally` ran; both may be nil -// and both run under mu. -func (r *router) sendFreshSfuConfig(participant authenticatedRouteParticipant, failed, settled func()) { - finish := func(err error) { - if err != nil && failed != nil { - failed() - } - if settled != nil { - settled() - } - } - fallback := r.sfu +// Reauthentication describes the existing physical connection without replacing it. +func (r *router) sendFreshSfuConfig(participant authenticatedRouteParticipant) { rm, _ := r.rooms.Get(participant.roomID) - shareGeneration := r.hooks.shareGeneration(participant.roomID) - if fallback == nil || rm == nil || rm.controller == nil { - finish(nil) + if r.sfu == nil || rm == nil || rm.controller == nil { return } snapshot := rm.controller.Snapshot() publication := snapshot.HostPublication - if publication == nil || !publication.PhysicalActive || shareGeneration == "" { - finish(nil) + if publication == nil || !publication.PhysicalActive { return } - isHost := participant.peerID == rm.hostPeerID - if !isHost { + connectionID := publication.ConnectionID + if participant.role == protocol.RoleViewer { edge, ok := snapshot.UpstreamByViewer.Get(participant.peerID) if !ok || edge.Kind != route.UpstreamSfu || !edge.PhysicalActive { - finish(nil) return } + connectionID = edge.ConnectionID } - role := protocol.RoleViewer - if isHost { - role = protocol.RoleHost - } - request := sfu.TokenRequest{ - RoomID: participant.roomID, - Role: role, - PeerID: participant.peerID, - ShareGeneration: shareGeneration, - PublicationGeneration: publication.Generation, - } - r.inflight++ - go func() { - token, err := fallback.tokenIssuer.IssueToken(request) - r.mu.Lock() - defer r.mu.Unlock() - r.inflight-- - if err != nil { - finish(err) - return - } - if rm.controller == nil { - finish(nil) - return - } - currentPublication := rm.controller.Publication() - peer, connected := r.connectedPeer(participant.roomID, participant.peerID) - if rm.controller.Revision() != snapshot.Revision || - currentPublication == nil || - currentPublication.Generation != publication.Generation || - !connected || peer.SessionID != participant.sessionID { - finish(nil) - return - } - r.hooks.sendToSession(participant.sessionID, protocol.SfuConfigMessage{ - Type: "sfu-config", - Revision: protocol.Int(snapshot.Revision), - URL: fallback.url, - Token: token, - }) - finish(nil) - }() -} - -// activeHostPublication ports activeHostPublication. -func (r *router) activeHostPublication(roomID string) (hostOfflineCheck, bool) { - rm, _ := r.rooms.Get(roomID) - if rm == nil || rm.controller == nil { - return hostOfflineCheck{}, false - } - publication := rm.controller.Publication() - if publication == nil || !publication.PhysicalActive || - publication.Resource == nil || publication.Resource.Kind != route.ResourceSfuPublication { - return hostOfflineCheck{}, false - } - return hostOfflineCheck{ - fence: resourceFence(publication.Resource), - hostSessionID: publication.HostSessionID, - connectionID: publication.ConnectionID, - }, true -} - -// scheduleHostOfflineCheck ports scheduleHostOfflineCheck (T7). -func (r *router) scheduleHostOfflineCheck(roomID, hostPeerID string) { - r.cancelHostOfflineCheck(roomID) - fallback := r.sfu - publication, ok := r.activeHostPublication(roomID) - if fallback == nil || !ok || r.closing { - return - } - check := &publication - check.hostPeerID = hostPeerID - delay := fallback.hostOfflineCheckMs - if delay == 0 { - delay = defaultHostOfflineCheckMs - } - check.stop = r.afterFunc(time.Duration(delay)*time.Millisecond, func() { - r.checkHostOffline(roomID, check) + r.hooks.sendToSession(participant.sessionID, protocol.SfuConfigMessage{ + Type: "sfu-config", Revision: protocol.Int(snapshot.Revision), + PublicationGeneration: publication.Generation, ConnectionID: connectionID, }) - r.hostOfflineChecks.Set(roomID, check) -} - -// checkHostOffline ports checkHostOffline (R12) as the timer callback body. -// armed is the check the timer was armed with: a cleared TS timer never -// fired, so anything else registered for the room means return (D5). -func (r *router) checkHostOffline(roomID string, armed *hostOfflineCheck) { - r.mu.Lock() - defer r.mu.Unlock() - check, _ := r.hostOfflineChecks.Get(roomID) - fallback := r.sfu - if check == nil || check != armed || fallback == nil || r.closing { - return - } - if _, hostConnected := r.store.GetConnectedHost(roomID); hostConnected { - r.cancelHostOfflineCheck(roomID) - return - } - var exists bool - var err error - r.io(func() { - exists, err = fallback.roomControl.HostParticipantExists(context.Background(), check.fence) - }) - if err != nil { - if registered, _ := r.hostOfflineChecks.Get(roomID); registered != check { - return + if participant.role == protocol.RoleHost && publication.Resource != nil { + if active, err := r.sfu.media.PublicationDemand(resourceFence(publication.Resource), connectionID); err == nil { + count := protocol.Int(active) + r.hooks.sendToSession(participant.sessionID, protocol.SfuSignalMessage{ + Type: "sfu-signal", Kind: "layers", Revision: protocol.Int(snapshot.Revision), + PublicationGeneration: publication.Generation, ConnectionID: connectionID, ActiveCount: &count, + }) } - r.hostOfflineChecks.Delete(roomID) - r.scheduleHostOfflineCheck(roomID, check.hostPeerID) - return } - if registered, _ := r.hostOfflineChecks.Get(roomID); registered != check { - return - } - r.hostOfflineChecks.Delete(roomID) - if exists { - r.scheduleHostOfflineCheck(roomID, check.hostPeerID) - return - } - rm, _ := r.rooms.Get(roomID) - if rm == nil || rm.controller == nil { - return - } - controller := rm.controller - snapshot := controller.Snapshot() - publication := snapshot.HostPublication - if publication == nil { - return - } - // Hazard 9: six publication fields plus the fence triple; anything - // short of that re-arms the check instead of retiring the publication. - if publication.Resource == nil || - publication.Resource.Kind != route.ResourceSfuPublication || - resourceFence(publication.Resource) != check.fence || - publication.Generation != check.fence.PublicationGeneration || - publication.HostSessionID != check.hostSessionID || - publication.ConnectionID != check.connectionID || - !publication.PhysicalActive { - r.scheduleHostOfflineCheck(roomID, check.hostPeerID) - return - } - beforeRevision := snapshot.Revision - controller.InvalidateHostPublication(route.PublicationGuard{ - HostSessionID: publication.HostSessionID, - RouteRevision: snapshot.Revision, - Generation: publication.Generation, - ConnectionID: publication.ConnectionID, - }, r.nowPtr()) - r.releaseResources(controller.RetireHostPublication(route.PublicationGuard{ - HostSessionID: publication.HostSessionID, - RouteRevision: controller.Revision(), - Generation: publication.Generation, - ConnectionID: publication.ConnectionID, - })) - afterRetirement := controller.Publication() - if afterRetirement != nil && - afterRetirement.Generation == publication.Generation && - afterRetirement.ConnectionID == publication.ConnectionID && - afterRetirement.PhysicalActive { - if controller.Revision() != beforeRevision { - r.broadcastActive(roomID, rm) - } - r.scheduleHostOfflineCheck(roomID, check.hostPeerID) - r.requestPump(roomID) - return - } - r.broadcastActive(roomID, rm) - r.requestPump(roomID) -} - -func (r *router) cancelHostOfflineCheck(roomID string) { - if check, ok := r.hostOfflineChecks.Get(roomID); ok { - check.stop() - } - r.hostOfflineChecks.Delete(roomID) } // scheduleSfuPublicationDrain ports scheduleSfuPublicationDrain (O28: the // sibling subscription tasks are deleted during the live iteration). func (r *router) scheduleSfuPublicationDrain(fence sfu.ResourceFence) { - roomName := sfu.ManagedRoomName(fence) for key, task := range r.sfuDrainTasks.All() { - if task.kind == drainSubscription && sfu.ManagedRoomName(task.fence.ResourceFence) == roomName { - r.clearDrainRetry(task) + if task.kind == drainSubscription && task.fence.ResourceFence == fence { r.sfuDrainTasks.Delete(key) } } @@ -420,7 +167,7 @@ func (r *router) scheduleSfuPublicationDrain(fence sfu.ResourceFence) { // scheduleSfuSubscriptionDrain ports scheduleSfuSubscriptionDrain: a pending // publication drain of the same room already removes every participant. func (r *router) scheduleSfuSubscriptionDrain(fence sfu.SubscriptionFence) { - if r.sfuDrainTasks.Has(sfu.ManagedRoomName(fence.ResourceFence)) { + if r.sfuDrainTasks.Has(sfu.SubscriptionFence{ResourceFence: fence.ResourceFence}) { return } r.scheduleSfuDrain(drainSubscription, fence) @@ -431,23 +178,20 @@ func (r *router) scheduleSfuDrain(kind drainKind, fence sfu.SubscriptionFence) { if r.sfu == nil { return } - key := sfuDrainKey(kind, fence) + key := fence task, _ := r.sfuDrainTasks.Get(key) if task == nil { task = &sfuDrainTask{kind: kind, fence: fence} r.sfuDrainTasks.Set(key, task) } - if task.operation == nil && task.retryStop == nil { + if task.operation == nil { // TS: void this.runSfuDrain(key, task).catch(() => undefined) - r.runSfuDrain(r.drainContext(), key, task) + r.runSfuDrain(key, task) } } -// runSfuDrain ports runSfuDrain (R13, R14). mu must be held; the returned -// operation is the tracked promise. The LiveKit call runs in a goroutine -// inside the room's FIFO turn taken here, and the post-await body runs under -// mu before done is closed. -func (r *router) runSfuDrain(ctx context.Context, key string, task *sfuDrainTask) *drainOperation { +// runSfuDrain performs physical close outside the authority mutex. +func (r *router) runSfuDrain(key sfu.SubscriptionFence, task *sfuDrainTask) *drainOperation { if task.operation != nil { return task.operation } @@ -458,21 +202,17 @@ func (r *router) runSfuDrain(ctx context.Context, key string, task *sfuDrainTask return operation } task.operation = operation - turn := r.takeTurn(sfu.ManagedRoomName(task.fence.ResourceFence)) r.inflight++ go func() { var err error - turn.run(func() { - if task.kind == drainPublication { - err = fallback.roomControl.DeleteRoom(ctx, task.fence.ResourceFence) - } else { - err = fallback.roomControl.DrainSubscription(ctx, task.fence) - } - }) + if task.kind == drainPublication { + err = fallback.media.ClosePublication(task.fence.ResourceFence) + } else { + err = fallback.media.CloseSubscription(task.fence, task.fence.ConnectionID) + } r.mu.Lock() defer r.mu.Unlock() r.inflight-- - r.dropTurn(turn) operation.err = r.settleSfuDrain(key, task, err) if task.operation == operation { task.operation = nil @@ -482,67 +222,25 @@ func (r *router) runSfuDrain(ctx context.Context, key string, task *sfuDrainTask return operation } -// settleSfuDrain is the post-await body of runSfuDrain: the try/catch after -// the LiveKit call. It returns the error the tracked promise rejected with. -func (r *router) settleSfuDrain(key string, task *sfuDrainTask, err error) error { +// A failed physical close remains charged until shutdown completes it. +func (r *router) settleSfuDrain(key sfu.SubscriptionFence, task *sfuDrainTask, err error) error { fallback := r.sfu if err == nil { if registered, _ := r.sfuDrainTasks.Get(key); registered != task { return nil } if task.kind == drainPublication && !fallback.admission.CompleteDrain(task.fence.ResourceFence) { - err = errors.New("LiveKit drain has no matching resource generation") + err = errors.New("SFU drain has no matching resource generation") } else { - r.sfuDrainTasks.Delete(key) - if task.kind == drainPublication { - r.wakeResourceWaiters() - } else { - r.requestPump(task.fence.RoomID) + if task.kind == drainSubscription { + fallback.admission.CompleteSubscriptionDrain(task.fence) } + r.sfuDrainTasks.Delete(key) + r.wakeResourceWaiters() return nil } } - // catch - if registered, _ := r.sfuDrainTasks.Get(key); registered != task { - return nil - } - if r.closing { - return err - } - retryMs := fallback.drainRetryMs - if retryMs == 0 { - retryMs = defaultSfuDrainRetryMs - } - generation := task.retryGeneration - task.retryStop = r.afterFunc(time.Duration(retryMs)*time.Millisecond, func() { // T8 - r.mu.Lock() - defer r.mu.Unlock() - if task.retryGeneration != generation || r.closing { - return - } - task.retryStop = nil - r.runSfuDrain(r.drainContext(), key, task) - }) - return nil -} - -// drainContext bounds the LiveKit drains: unbounded (the TS had none) until -// close installs its context (D6). -func (r *router) drainContext() context.Context { - if r.closeCtx != nil { - return r.closeCtx - } - return context.Background() -} - -// clearDrainRetry is the clearTimeout(task.retryTimer) of the TS; bumping -// the generation retires a callback that already lost the Stop race. -func (r *router) clearDrainRetry(task *sfuDrainTask) { - if task.retryStop != nil { - task.retryStop() - task.retryStop = nil - } - task.retryGeneration++ + return err } // awaitDrain is `await task.operation` inside close(): mu is released while @@ -560,14 +258,6 @@ func (r *router) awaitDrain(ctx context.Context, operation *drainOperation) erro return err } -func sfuDrainKey(kind drainKind, fence sfu.SubscriptionFence) string { - roomName := sfu.ManagedRoomName(fence.ResourceFence) - if kind == drainPublication { - return roomName - } - return roomName + "\x00viewer:" + fence.ViewerPeerID -} - // reservationResources ports reservationResources: a borrowed reuse edge // belongs to another holder and is not released. func reservationResources(reservation route.CandidateReservation) []*route.Resource { diff --git a/internal/server/signal/router_sfu_signal.go b/internal/server/signal/router_sfu_signal.go new file mode 100644 index 0000000..27ea83d --- /dev/null +++ b/internal/server/signal/router_sfu_signal.go @@ -0,0 +1,254 @@ +package signal + +import ( + "context" + "encoding/json" + "time" + + "github.com/TNTcraftHIM/Screener/internal/media/forwarding" + "github.com/TNTcraftHIM/Screener/internal/server/protocol" + "github.com/TNTcraftHIM/Screener/internal/server/route" + "github.com/TNTcraftHIM/Screener/internal/server/sfu" + "github.com/pion/webrtc/v4" +) + +func (r *router) sfuSignalAuthorized(participant authenticatedRouteParticipant, message protocol.SfuSignalMessage) (sfu.SubscriptionFence, bool) { + fence := sfu.SubscriptionFence{ResourceFence: sfu.ResourceFence{ + RoomID: participant.roomID, ShareGeneration: r.hooks.shareGeneration(participant.roomID), + PublicationGeneration: message.PublicationGeneration, + }, ConnectionID: message.ConnectionID} + rm, _ := r.rooms.Get(participant.roomID) + peer, connected := r.connectedPeer(participant.roomID, participant.peerID) + if r.closing || r.sfu == nil || rm == nil || rm.controller == nil || !connected || peer.SessionID != participant.sessionID { + return fence, false + } + snapshot := rm.controller.Snapshot() + latestRevision := snapshot.Revision + if snapshot.Operation != nil && snapshot.Operation.Current != nil { + latestRevision = max(latestRevision, snapshot.Operation.Current.Revision) + } + if int64(message.Revision) > latestRevision { + return fence, false + } + if participant.role == protocol.RoleHost { + publication := snapshot.HostPublication + if publication != nil && publication.PhysicalActive && publication.Generation == message.PublicationGeneration && + publication.ConnectionID == message.ConnectionID && publication.HostSessionID == participant.sessionID { + return fence, r.sfu.admission.HasPublication(fence.ResourceFence) + } + operation := snapshot.Operation + return fence, participant.peerID == rm.hostPeerID && operation != nil && operation.Current != nil && + operation.Current.Tuple.Kind == route.UpstreamSfu && operation.Current.Tuple.Publication != route.PublicationReuse && + operation.Current.Revision == int64(message.Revision) && message.ConnectionID == message.PublicationGeneration && + r.sfu.admission.HasPublication(fence.ResourceFence) + } + fence.ViewerPeerID = participant.peerID + if !r.sfu.admission.HasSubscription(fence) { + return fence, false + } + if edge, ok := snapshot.UpstreamByViewer.Get(participant.peerID); ok && edge.PhysicalActive && edge.Kind == route.UpstreamSfu && + edge.PublicationGeneration == message.PublicationGeneration && edge.ConnectionID == message.ConnectionID && edge.ChildSessionID == participant.sessionID { + return fence, true + } + operation := snapshot.Operation + return fence, operation != nil && operation.Current != nil && operation.ChildPeerID == participant.peerID && + operation.ChildSessionID == participant.sessionID && operation.Current.Tuple.Kind == route.UpstreamSfu && + operation.Current.Revision == int64(message.Revision) && operation.Current.ConnectionID == message.ConnectionID +} + +func (r *router) handleSfuSignal(participant authenticatedRouteParticipant, message protocol.SfuSignalMessage) { + fence, allowed := r.sfuSignalAuthorized(participant, message) + if !allowed { + return + } + media := r.sfu.media + // Each session's input loop serializes SDP/ICE. Preparation itself is bounded + // by the route operation and runs outside the authority mutex. + var description webrtc.SessionDescription + var err error + newPublication := false + r.io(func() { + switch message.Kind { + case "media": + if participant.role == protocol.RoleHost { + err = media.UpdatePublication(fence.ResourceFence, message.ConnectionID, sfuPublicationMedia(message.Media)) + } + case "description": + incoming := webrtc.SessionDescription{Type: webrtc.NewSDPType(message.Description.Type), SDP: message.Description.SDP} + if participant.role == protocol.RoleHost { + if incoming.Type != webrtc.SDPTypeOffer { + return + } + if !media.HasPublication(fence.ResourceFence, message.ConnectionID) { + if message.Media == nil { + return + } + metadata := sfuPublicationMedia(message.Media) + err = media.PreparePublication(fence.ResourceFence, message.ConnectionID, metadata) + newPublication = err == nil + } + if err == nil { + description, err = media.AcceptPublisherOffer(fence.ResourceFence, message.ConnectionID, incoming) + } + } else if incoming.Type == webrtc.SDPTypeAnswer { + err = media.ApplySubscriberAnswer(fence, message.ConnectionID, incoming) + } + case "candidate": + var candidate webrtc.ICECandidateInit + if message.Candidate != nil { + encoded, _ := json.Marshal(message.Candidate) + _ = json.Unmarshal(encoded, &candidate) + } + if participant.role == protocol.RoleHost { + err = media.AddPublisherICE(fence.ResourceFence, message.ConnectionID, candidate) + } else { + err = media.AddSubscriberICE(fence, message.ConnectionID, candidate) + } + case "subscribe": + if participant.role == protocol.RoleViewer { + description, err = media.RestartSubscriber(fence, message.ConnectionID) + } + } + }) + if _, allowed := r.sfuSignalAuthorized(participant, message); !allowed { + if newPublication { + go media.ClosePublication(fence.ResourceFence) + } + return + } + if err != nil { + r.failSfuConnection(participant, message) + return + } + if description.SDP != "" { + r.sendSfuDescription(participant.sessionID, message, description) + } + if newPublication { + rm, _ := r.rooms.Get(participant.roomID) + operation := rm.controller.Operation() + if operation != nil && operation.Current != nil && operation.Current.Tuple.Kind == route.UpstreamSfu { + r.prepareSfuSubscriber(participant.roomID, operation, message.PublicationGeneration) + } + } +} + +func sfuPublicationMedia(value *protocol.SfuMedia) sfu.PublicationMedia { + metadata := sfu.PublicationMedia{Codec: value.Codec, Audio: value.Audio, AudioBitrate: uint32(value.AudioBitrate)} + for _, layer := range value.Layers { + metadata.RIDs = append(metadata.RIDs, layer.RID) + metadata.Formats = append(metadata.Formats, forwarding.LayerFormat{Width: uint32(layer.Width), Height: uint32(layer.Height), Bitrate: uint32(layer.Bitrate)}) + } + return metadata +} + +func (r *router) prepareSfuSubscriber(roomID string, operation *route.OperationSnapshot, generation string) { + message := protocol.SfuSignalMessage{Type: "sfu-signal", Kind: "description", Revision: protocol.Int(operation.Current.Revision), PublicationGeneration: generation, ConnectionID: operation.Current.ConnectionID} + participant := authenticatedRouteParticipant{roomID: roomID, role: protocol.RoleViewer, peerID: operation.ChildPeerID, sessionID: operation.ChildSessionID} + fence, allowed := r.sfuSignalAuthorized(participant, message) + if !allowed { + return + } + media := r.sfu.media + ctx, cancel := context.WithTimeout(context.Background(), time.Duration(max(int64(0), operation.DeadlineAtMs-r.now()))*time.Millisecond) + r.inflight++ + go func() { + defer cancel() + description, err := media.PrepareSubscriber(ctx, fence, message.ConnectionID) + r.mu.Lock() + defer r.mu.Unlock() + r.inflight-- + if _, allowed := r.sfuSignalAuthorized(participant, message); !allowed { + go media.CloseSubscription(fence, message.ConnectionID) + return + } + if err != nil { + r.failSfuConnection(participant, message) + return + } + r.sendSfuDescription(participant.sessionID, message, description) + }() +} + +func (r *router) sendSfuDescription(sessionID string, message protocol.SfuSignalMessage, description webrtc.SessionDescription) { + message.Kind = "description" + message.Description = &protocol.SessionDescription{Type: description.Type.String(), SDP: description.SDP} + message.Media, message.Candidate = nil, nil + r.hooks.sendToSession(sessionID, message) +} + +func (r *router) failSfuConnection(participant authenticatedRouteParticipant, message protocol.SfuSignalMessage) { + rm, _ := r.rooms.Get(participant.roomID) + if rm == nil || rm.controller == nil { + return + } + connectionID := message.ConnectionID + snapshot := rm.controller.Snapshot() + phase, revision := sfuConnectionPhase(snapshot, participant.role, participant.peerID, message.PublicationGeneration, connectionID) + if phase == "prepare" { + connectionID = snapshot.Operation.Current.ConnectionID + } + r.handleRouteFailed(participant, protocol.RouteFailedMessage{Type: "route-failed", Revision: protocol.Int(revision), Phase: phase, ConnectionID: &connectionID}) +} + +func sfuConnectionPhase(snapshot route.RouteSnapshot, role protocol.Role, peerID, generation, connectionID string) (string, int64) { + operation := snapshot.Operation + if operation != nil && operation.Current != nil && operation.Current.Tuple.Kind == route.UpstreamSfu && + (role == protocol.RoleViewer && operation.ChildPeerID == peerID && operation.Current.ConnectionID == connectionID || + role == protocol.RoleHost && connectionID == generation && (snapshot.HostPublication == nil || snapshot.HostPublication.Generation != generation)) { + return "prepare", operation.Current.Revision + } + return "active", snapshot.Revision +} + +// Media callbacks identify physical handles; signaling reconnect does not retire them. +func (s *Server) HandleSfuMediaEvent(event sfu.MediaEvent) { + s.mu.Lock() + defer s.mu.Unlock() + s.router.handleSfuMediaEvent(event) +} + +func (r *router) handleSfuMediaEvent(event sfu.MediaEvent) { + rm, _ := r.rooms.Get(event.Fence.RoomID) + if r.closing || rm == nil || rm.controller == nil || r.hooks.shareGeneration(event.Fence.RoomID) != event.Fence.ShareGeneration { + return + } + peerID, role := event.ViewerPeerID, protocol.RoleViewer + if peerID == "" { + peerID, role = rm.hostPeerID, protocol.RoleHost + } + peer, connected := r.connectedPeer(event.Fence.RoomID, peerID) + snapshot := rm.controller.Snapshot() + participant := authenticatedRouteParticipant{roomID: event.Fence.RoomID, role: role, peerID: peerID, sessionID: peer.SessionID} + _, revision := sfuConnectionPhase(snapshot, role, peerID, event.Fence.PublicationGeneration, event.ConnectionID) + message := protocol.SfuSignalMessage{Type: "sfu-signal", Kind: "candidate", Revision: protocol.Int(revision), PublicationGeneration: event.Fence.PublicationGeneration, ConnectionID: event.ConnectionID} + if !connected && role == protocol.RoleHost && event.State == webrtc.PeerConnectionStateFailed { + publication := snapshot.HostPublication + if publication != nil && publication.Generation == event.Fence.PublicationGeneration && publication.ConnectionID == event.ConnectionID { + participant.sessionID = publication.HostSessionID + r.failSfuConnection(participant, message) + } + return + } + if _, allowed := r.sfuSignalAuthorized(participant, message); !allowed { + return + } + if event.ActiveCount != nil { + message.Kind = "layers" + count := protocol.Int(*event.ActiveCount) + message.ActiveCount = &count + r.hooks.sendToSession(participant.sessionID, message) + return + } + if event.State == webrtc.PeerConnectionStateFailed { + r.failSfuConnection(participant, message) + return + } + if event.State != webrtc.PeerConnectionStateUnknown { + return + } + if event.Candidate != nil { + encoded, _ := json.Marshal(event.Candidate) + _ = json.Unmarshal(encoded, &message.Candidate) + } + r.hooks.sendToSession(participant.sessionID, message) +} diff --git a/internal/server/signal/router_sfu_signal_test.go b/internal/server/signal/router_sfu_signal_test.go new file mode 100644 index 0000000..baee85d --- /dev/null +++ b/internal/server/signal/router_sfu_signal_test.go @@ -0,0 +1,299 @@ +package signal + +import ( + "context" + "encoding/json" + "net" + "sync/atomic" + "testing" + "time" + + "github.com/TNTcraftHIM/Screener/internal/server/protocol" + "github.com/TNTcraftHIM/Screener/internal/server/sfu" + livekitsfu "github.com/livekit/livekit-server/pkg/sfu" + "github.com/pion/webrtc/v4" + "github.com/pion/webrtc/v4/pkg/media" +) + +func TestEmbeddedSfuSignalsRequireExactAdmittedConnectionAndCurrentSession(t *testing.T) { + h := newRouterHarness(t, routerHarnessOptions{capacity: 1, withSfu: true}) + created := h.createRoom() + host, first, second := h.establishSfuRoom(created) + h.settle() + h.locked(func() { + rm, _ := h.router.rooms.Get(created.RoomID) + publication := rm.controller.Publication() + edge := h.doMustEdge(created.RoomID, first.peerID) + message := protocol.SfuSignalMessage{Type: "sfu-signal", Kind: "candidate", Revision: protocol.Int(edge.revision), PublicationGeneration: publication.Generation, ConnectionID: edge.connectionID} + if _, ok := h.router.sfuSignalAuthorized(first, message); !ok { + t.Fatal("current subscriber was rejected") + } + if _, ok := h.router.sfuSignalAuthorized(second, message); ok { + t.Fatal("another viewer could signal the subscription") + } + stale := first + stale.sessionID = "stale_session_12345678" + if _, ok := h.router.sfuSignalAuthorized(stale, message); ok { + t.Fatal("replaced control session retained authority") + } + forged := message + forged.Revision += 100 + if _, ok := h.router.sfuSignalAuthorized(first, forged); ok { + t.Fatal("future route revision was accepted") + } + forged = message + forged.PublicationGeneration = "unowned_publication_12345678" + if _, ok := h.router.sfuSignalAuthorized(first, forged); ok { + t.Fatal("unowned publication was accepted") + } + message.ConnectionID = publication.ConnectionID + if _, ok := h.router.sfuSignalAuthorized(first, message); ok { + t.Fatal("viewer could signal Host ingress") + } + message.Kind = "description" + message.Description = &protocol.SessionDescription{Type: "offer", SDP: "test-offer"} + message.Media = &protocol.SfuMedia{Codec: "vp8", Layers: []protocol.SfuLayer{{RID: "", Width: 640, Height: 360, Bitrate: 500_000}}} + h.router.handleSfuSignal(host, message) + if len(h.media.Created()) != 1 { + t.Fatal("authorized publisher did not create media") + } + message.ConnectionID = "stale_connection_12345678" + h.router.handleSfuSignal(host, message) + if len(h.media.Created()) != 1 { + t.Fatal("stale publisher created another physical handle") + } + }) +} + +func TestEmbeddedSfuAuthenticatedWebSocketCarriesActualRTP(t *testing.T) { + ctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + check := func(err error) { + t.Helper() + if err != nil { + t.Fatal(err) + } + } + settings := webrtc.SettingEngine{} + settings.SetNetworkTypes([]webrtc.NetworkType{webrtc.NetworkTypeUDP4}) + settings.SetIncludeLoopbackCandidate(true) + settings.SetIPFilter(func(ip net.IP) bool { return ip.IsLoopback() }) + var signaling atomic.Pointer[Server] + service := sfu.NewMedia(sfu.MediaOptions{Settings: settings, Events: func(event sfu.MediaEvent) { + if server := signaling.Load(); server != nil { + server.HandleSfuMediaEvent(event) + } + }}) + defer service.Close() + h := startHarness(t, harnessOptions{sfu: &SfuFallback{Media: service, + Admission: sfu.NewAdmission(sfu.AdmissionOptions{IngressCapacity: 1, EgressCapacity: 2})}}) + signaling.Store(h.server) + defer h.close() + host, viewer := openClient(t, h), openClient(t, h) + authenticate(t, host, h.room, protocol.RoleHost, "embedded_host_12345678", 1, "", presenceOptions{}) + authenticate(t, viewer, h.room, protocol.RoleViewer, "embedded_viewer_12345678", 1, "", presenceOptions{}) + direct := nextPreparedRoute(t, viewer) + viewer.sendJSON(map[string]any{"type": "route-failed", "revision": direct.Revision, "phase": "prepare", "connectionId": direct.Candidate.ConnectionID}) + var hostConfig, viewerConfig protocol.SfuConfigMessage + check(json.Unmarshal(host.next("sfu-config").raw, &hostConfig)) + check(json.Unmarshal(viewer.next("sfu-config").raw, &viewerConfig)) + if hostConfig.PublicationGeneration != viewerConfig.PublicationGeneration || hostConfig.ConnectionID == viewerConfig.ConnectionID { + t.Fatal("publication and subscription fences were not distinct") + } + newPeer := func() *webrtc.PeerConnection { + pc, err := webrtc.NewAPI(webrtc.WithSettingEngine(settings)).NewPeerConnection(webrtc.Configuration{}) + check(err) + t.Cleanup(func() { _ = pc.Close() }) + return pc + } + gather := func(pc *webrtc.PeerConnection, description webrtc.SessionDescription) webrtc.SessionDescription { + done := webrtc.GatheringCompletePromise(pc) + check(pc.SetLocalDescription(description)) + select { + case <-done: + case <-ctx.Done(): + t.Fatal(ctx.Err()) + } + return *pc.LocalDescription() + } + sendDescription := func(client *testClient, configuration protocol.SfuConfigMessage, description webrtc.SessionDescription, declaration *protocol.SfuMedia) { + message := protocol.SfuSignalMessage{Type: "sfu-signal", Kind: "description", Revision: configuration.Revision, + PublicationGeneration: configuration.PublicationGeneration, ConnectionID: configuration.ConnectionID, + Description: &protocol.SessionDescription{Type: description.Type.String(), SDP: description.SDP}, Media: declaration} + encoded, err := json.Marshal(message) + check(err) + client.sendText(string(encoded)) + } + remoteCandidates := make(map[*webrtc.PeerConnection]int) + applyRemote := func(client *testClient, pc *webrtc.PeerConnection, configuration protocol.SfuConfigMessage, untilConnected bool) { + var candidates []webrtc.ICECandidateInit + for { + if ctx.Err() != nil { + t.Fatal(ctx.Err()) + } + if untilConnected && pc.ConnectionState() == webrtc.PeerConnectionStateConnected && remoteCandidates[pc] > 0 { + return + } + incoming, ok := client.tryNext("sfu-signal", 20*time.Millisecond) + if !ok { + continue + } + var message protocol.SfuSignalMessage + check(json.Unmarshal(incoming.raw, &message)) + if message.ConnectionID != configuration.ConnectionID || message.PublicationGeneration != configuration.PublicationGeneration || message.Revision != configuration.Revision { + t.Fatal("media signal lost its fence") + } + if message.Kind == "layers" { + continue + } + if message.Kind == "candidate" { + candidate := webrtc.ICECandidateInit{} + if message.Candidate != nil { + remoteCandidates[pc]++ + raw, err := json.Marshal(message.Candidate) + check(err) + check(json.Unmarshal(raw, &candidate)) + } + if pc.RemoteDescription() == nil { + candidates = append(candidates, candidate) + } else { + check(pc.AddICECandidate(candidate)) + } + continue + } + check(pc.SetRemoteDescription(webrtc.SessionDescription{Type: webrtc.NewSDPType(message.Description.Type), SDP: message.Description.SDP})) + for _, candidate := range candidates { + check(pc.AddICECandidate(candidate)) + } + candidates = nil + if !untilConnected { + return + } + } + } + publisher := newPeer() + video, err := webrtc.NewTrackLocalStaticSample(webrtc.RTPCodecCapability{MimeType: webrtc.MimeTypeVP8, ClockRate: 90_000}, "video", "stream") + check(err) + _, err = publisher.AddTrack(video) + check(err) + offer, err := publisher.CreateOffer(nil) + check(err) + sendDescription(host, hostConfig, gather(publisher, offer), &protocol.SfuMedia{Codec: "vp8", Layers: []protocol.SfuLayer{{RID: "", Width: 8, Height: 8, Bitrate: 500_000}}}) + applyRemote(host, publisher, hostConfig, true) + stopFrames, framesStopped := make(chan struct{}), make(chan struct{}) + go func() { + defer close(framesStopped) + ticker := time.NewTicker(time.Second / 30) + defer ticker.Stop() + for { + select { + case <-ticker.C: + _ = video.WriteSample(media.Sample{Data: livekitsfu.VP8KeyFrame8x8, Duration: time.Second / 30}) + case <-stopFrames: + return + } + } + }() + defer func() { close(stopFrames); <-framesStopped }() + subscriber := newPeer() + var received atomic.Uint32 + subscriber.OnTrack(func(track *webrtc.TrackRemote, _ *webrtc.RTPReceiver) { + for { + packet, _, err := track.ReadRTP() + if err != nil { + return + } + if track.Kind() == webrtc.RTPCodecTypeVideo && len(packet.Payload) > 0 { + received.Add(1) + } + } + }) + applyRemote(viewer, subscriber, viewerConfig, false) + answer, err := subscriber.CreateAnswer(nil) + check(err) + sendDescription(viewer, viewerConfig, gather(subscriber, answer), nil) + applyRemote(viewer, subscriber, viewerConfig, true) + if !waitFor(t, 3*time.Second, func() bool { return received.Load() >= 3 }) { + t.Fatal("authenticated signaling did not produce forwarded video RTP") + } + h.locked(func() { + room, _ := h.server.router.rooms.Get(h.room.RoomID) + if room.controller.Operation() == nil { + t.Fatal("transport/RTP arrival committed without client decoded-frame proof") + } + }) + viewer.sendJSON(map[string]any{"type": "route-ready", "revision": viewerConfig.Revision, "phase": "prepare"}) + if !waitFor(t, time.Second, func() bool { + committed := false + h.locked(func() { + room, _ := h.server.router.rooms.Get(h.room.RoomID) + committed = room.controller.Operation() == nil && room.controller.Publication() != nil + }) + return committed + }) { + t.Fatal("the exact client media-ready proof did not commit") + } +} + +func TestEmbeddedSfuRejectsBadPublisherOfferAtPendingRevision(t *testing.T) { + h := newRouterHarness(t, routerHarnessOptions{capacity: 1, withSfu: true}) + service := sfu.NewMedia(sfu.MediaOptions{}) + defer service.Close() + h.router.sfu.media = service + created := h.createRoom() + host := h.connectHost(created, nil, "", "") + viewer := h.connectViewer(created, "candidate_viewer") + h.complete(host) + h.complete(viewer) + direct := h.waitPrepared(viewer.sessionID) + h.routeFailed(viewer, routeFailedMessage(int64(direct.Revision), "prepare", direct.Candidate.ConnectionID)) + prepared := h.waitPreparedTransport(viewer.sessionID, "sfu") + var configuration protocol.SfuConfigMessage + for _, message := range h.messages(host.sessionID) { + if value, ok := message.(protocol.SfuConfigMessage); ok { + configuration = value + } + } + h.locked(func() { + h.router.handleSfuSignal(host, protocol.SfuSignalMessage{Type: "sfu-signal", Kind: "description", Revision: configuration.Revision, + PublicationGeneration: configuration.PublicationGeneration, ConnectionID: configuration.ConnectionID, + Description: &protocol.SessionDescription{Type: "offer", SDP: "invalid"}, + Media: &protocol.SfuMedia{Codec: "vp8", Layers: []protocol.SfuLayer{{RID: "", Width: 640, Height: 360, Bitrate: 500_000}}}, + }) + }) + h.waitUsage(sfu.Usage{}) + if _, ok := h.activeAfter(viewer.sessionID, int64(prepared.Revision)); !ok { + t.Fatal("failed pending media did not broadcast a newer rollback revision") + } +} + +func TestEmbeddedSfuControlLossKeepsMediaUntilExactTransportFailure(t *testing.T) { + h := newRouterHarness(t, routerHarnessOptions{capacity: 1, withSfu: true}) + created := h.createRoom() + host, _, _ := h.establishSfuRoom(created) + h.settle() + var event sfu.MediaEvent + h.locked(func() { + rm, _ := h.router.rooms.Get(created.RoomID) + publication := rm.controller.Publication() + event = sfu.MediaEvent{Fence: resourceFence(publication.Resource), ConnectionID: publication.ConnectionID, State: webrtc.PeerConnectionStateFailed} + h.doDisconnect(host) + }) + h.settle() + if len(h.media.Deleted()) != 0 || h.usage().Ingress != 1 { + t.Fatal("control loss retired healthy media") + } + h.locked(func() { + stale := event + stale.ConnectionID = "stale_connection_12345678" + h.router.handleSfuMediaEvent(stale) + }) + if len(h.media.Deleted()) != 0 { + t.Fatal("stale media callback retired the current publication") + } + h.locked(func() { h.router.handleSfuMediaEvent(event) }) + h.waitUsage(sfu.Usage{}) + if len(h.media.Deleted()) != 1 { + t.Fatal("exact physical failure did not retire ingress") + } +} diff --git a/internal/server/signal/router_test.go b/internal/server/signal/router_test.go index 426d330..7540c86 100644 --- a/internal/server/signal/router_test.go +++ b/internal/server/signal/router_test.go @@ -2,7 +2,7 @@ package signal // Ported from tests/hybrid-media-router.test.ts (baseline b20fd88). The // harness mirrors harness(...) there: a memory room.Store, a real -// sfu.Admission, sfutest.FakeRoomControl behind a thin spy (the vi.spyOn +// sfu.Admission, sfutest.FakeMedia behind a thin spy (the vi.spyOn // cases), a fake token issuer returning "token-", recorded hooks and // the shared mutex every router call and every assertion takes. // @@ -21,6 +21,7 @@ package signal import ( "context" "errors" + "io" "log/slog" "reflect" "runtime" @@ -50,8 +51,6 @@ type routerHarnessOptions struct { withSfu bool prepareTimeoutMs int64 sfuIngressCapacity int - drainRetryMs int64 - hostOfflineCheckMs int64 fakeTimers bool } @@ -61,8 +60,7 @@ type routerHarness struct { store *room.Store router *router admission *sfu.Admission - roomControl *spyRoomControl - tokens *fakeTokenIssuer + media *spyMedia clock *manualClock sent map[string][]protocol.ServerMessage routesChanged []string @@ -92,16 +90,11 @@ func newRouterHarness(t *testing.T, options routerHarnessOptions) *routerHarness ingress = 2 } h.admission = sfu.NewAdmission(sfu.AdmissionOptions{IngressCapacity: ingress, EgressCapacity: 20}) - h.roomControl = &spyRoomControl{FakeRoomControl: sfutest.New()} - h.tokens = &fakeTokenIssuer{} + h.media = &spyMedia{FakeMedia: sfutest.New()} fallback = &sfuFallback{ - url: "wss://sfu.example.test", - tokenIssuer: h.tokens, - admission: h.admission, - roomControl: h.roomControl, - prepareTimeoutMs: options.prepareTimeoutMs, - drainRetryMs: options.drainRetryMs, - hostOfflineCheckMs: options.hostOfflineCheckMs, + admission: h.admission, + media: h.media, + prepareTimeoutMs: options.prepareTimeoutMs, } } h.router = newRouter(routerOptions{ @@ -503,18 +496,6 @@ func (h *routerHarness) usage() sfu.Usage { return usage } -func (h *routerHarness) failNextTokenIssue() { - h.tokens.mu.Lock() - defer h.tokens.mu.Unlock() - h.tokens.nextError = errors.New("token issue failed") -} - -func (h *routerHarness) panicNextTokenIssue() { - h.tokens.mu.Lock() - defer h.tokens.mu.Unlock() - h.tokens.nextPanic = true -} - // establishSfuRoom ports establishSfuRoom: a Host, a first Viewer moved to // the SFU and a second Viewer joining it. func (h *routerHarness) establishSfuRoom(created room.CreatedRoom) (host, first, second authenticatedRouteParticipant) { @@ -567,113 +548,32 @@ func senderEvidenceMessage(childPeerID, connectionID, rtpStatsID, trackIdentifie } } -// fakeTokenIssuer is the harness tokenIssuer: "token-" with a -// one-shot failure switch and a one-shot panic switch. It is called with the -// router mutex released. -type fakeTokenIssuer struct { - mu sync.Mutex - nextError error - nextPanic bool +type spyMedia struct { + *sfutest.FakeMedia + mu sync.Mutex + deleteCalls int + closePublication func(sfu.ResourceFence) error } -func (f *fakeTokenIssuer) IssueToken(request sfu.TokenRequest) (string, error) { - f.mu.Lock() - defer f.mu.Unlock() - if f.nextPanic { - f.nextPanic = false - panic(errors.New("token issue panicked")) - } - if f.nextError != nil { - err := f.nextError - f.nextError = nil - return "", err - } - return "token-" + request.PeerID, nil -} - -// spyRoomControl is the vi.spyOn(roomControl, ...) of the TS suite over the -// shared fake: call counters, one-shot behaviours (mock*Once) and a -// persistent deleteRoom implementation. -type spyRoomControl struct { - *sfutest.FakeRoomControl - mu sync.Mutex - deleteRoomCalls int - hostCheckCalls int - drainCalls []sfu.SubscriptionFence - deleteRoom func(context.Context, sfu.ResourceFence) error - hostCheckOnce []func(context.Context, sfu.ResourceFence) (bool, error) - drainOnce []func(context.Context, sfu.SubscriptionFence) error -} - -func (s *spyRoomControl) DeleteRoom(ctx context.Context, fence sfu.ResourceFence) error { - s.mu.Lock() - s.deleteRoomCalls++ - override := s.deleteRoom - s.mu.Unlock() +func (media *spyMedia) ClosePublication(fence sfu.ResourceFence) error { + media.mu.Lock() + media.deleteCalls++ + override := media.closePublication + media.mu.Unlock() if override != nil { - return override(ctx, fence) + return override(fence) } - return s.FakeRoomControl.DeleteRoom(ctx, fence) + return media.FakeMedia.ClosePublication(fence) } - -func (s *spyRoomControl) HostParticipantExists(ctx context.Context, fence sfu.ResourceFence) (bool, error) { - s.mu.Lock() - s.hostCheckCalls++ - var once func(context.Context, sfu.ResourceFence) (bool, error) - if len(s.hostCheckOnce) > 0 { - once, s.hostCheckOnce = s.hostCheckOnce[0], s.hostCheckOnce[1:] - } - s.mu.Unlock() - if once != nil { - return once(ctx, fence) - } - return s.FakeRoomControl.HostParticipantExists(ctx, fence) +func (media *spyMedia) counts() (int, int, int) { + media.mu.Lock() + defer media.mu.Unlock() + return media.deleteCalls, 0, len(media.SubscriptionDrainAttempts()) } - -func (s *spyRoomControl) DrainSubscription(ctx context.Context, fence sfu.SubscriptionFence) error { - s.mu.Lock() - s.drainCalls = append(s.drainCalls, fence) - var once func(context.Context, sfu.SubscriptionFence) error - if len(s.drainOnce) > 0 { - once, s.drainOnce = s.drainOnce[0], s.drainOnce[1:] - } - s.mu.Unlock() - if once != nil { - return once(ctx, fence) - } - return s.FakeRoomControl.DrainSubscription(ctx, fence) -} - -func (s *spyRoomControl) counts() (deleteRoom, hostCheck, drain int) { - s.mu.Lock() - defer s.mu.Unlock() - return s.deleteRoomCalls, s.hostCheckCalls, len(s.drainCalls) -} - -// drainAttempts is the spy's own record (the TS toHaveBeenCalledWith on the -// spied drainSubscription), which also sees the mocked calls. -func (s *spyRoomControl) drainAttempts() []sfu.SubscriptionFence { - s.mu.Lock() - defer s.mu.Unlock() - return slices.Clone(s.drainCalls) -} - -func (s *spyRoomControl) queueHostCheck(behaviours ...func(context.Context, sfu.ResourceFence) (bool, error)) { - s.mu.Lock() - defer s.mu.Unlock() - s.hostCheckOnce = append(s.hostCheckOnce, behaviours...) -} - -func (s *spyRoomControl) queueDrain(behaviours ...func(context.Context, sfu.SubscriptionFence) error) { - s.mu.Lock() - defer s.mu.Unlock() - s.drainOnce = append(s.drainOnce, behaviours...) -} - -func (s *spyRoomControl) setDeleteRoom(implementation func(context.Context, sfu.ResourceFence) error) { - s.mu.Lock() - defer s.mu.Unlock() - s.deleteRoom = implementation +func (media *spyMedia) setClosePublication(closePublication func(sfu.ResourceFence) error) { + media.mu.Lock() + defer media.mu.Unlock() + media.closePublication = closePublication } func hasViewer(fences []sfu.SubscriptionFence, viewerPeerID string) bool { @@ -783,13 +683,11 @@ type debugRecorder struct { func captureRouteDebug(t *testing.T) *debugRecorder { t.Helper() recorder := &debugRecorder{} - previousEnabled := routeDebugEnabled previousLogger := slog.Default() - routeDebugEnabled = true + t.Setenv("SCREENER_DEBUG", "route") slog.SetDefault(slog.New(recorder)) t.Cleanup(func() { slog.SetDefault(previousLogger) - routeDebugEnabled = previousEnabled }) return recorder } @@ -1670,8 +1568,8 @@ func TestRouterDoesNotRecreateSfuAfterExhaustedReuseAndCarrierTeardown(t *testin h.routeReady(carrier, int64(bootstrap.Revision)) h.waitPreparedTransport(demand.sessionID, "sfu") - if len(h.roomControl.Created()) != 1 { - t.Fatalf("created rooms = %d, want 1", len(h.roomControl.Created())) + if h.usage().Ingress != 1 { + t.Fatal("expected one admitted publication") } configCountAfterBootstrap := sfuConfigCount(h.allMessages()) if configCountAfterBootstrap < 3 { @@ -1693,11 +1591,11 @@ func TestRouterDoesNotRecreateSfuAfterExhaustedReuseAndCarrierTeardown(t *testin edge, ok := h.activeEdge(created.RoomID, carrier.peerID) return ok && edge.upstream == protocol.PeerUpstream(host.peerID) }) - h.waitFor("the publication drain", func() bool { return len(h.roomControl.Deleted()) == 1 }) + h.waitFor("the publication drain", func() bool { return len(h.media.Deleted()) == 1 }) h.clock.advance(300) - if len(h.roomControl.Created()) != 1 { - t.Fatalf("created rooms = %d, want 1", len(h.roomControl.Created())) + if h.usage().Ingress != 0 { + t.Fatal("retired publication remained admitted") } if count := sfuConfigCount(h.allMessages()); count != configCountAfterBootstrap { t.Fatalf("sfu-config count = %d, want %d", count, configCountAfterBootstrap) @@ -1850,7 +1748,7 @@ func TestRouterReportsOnlySfuAdmissionWaitingWhileBoundedCapacityIsUnavailable(t // TS asserted admission.reservePublication was not called again through a // spy; here the observable is the waiting status count, which a woken // waiter would re-send (map 7.2 step 9c), plus the unchanged usage. -func TestRouterPhysicallyRemovesExactSubscriptionWithoutReleasingGenerationCharge(t *testing.T) { +func TestRouterReleasesExactSubscriptionChargeAfterPhysicalClose(t *testing.T) { h := newRouterHarness(t, routerHarnessOptions{capacity: 1, withSfu: true, sfuIngressCapacity: 2}) activeRoom := h.createRoom() _, first, _ := h.establishSfuRoom(activeRoom) @@ -1871,9 +1769,8 @@ func TestRouterPhysicallyRemovesExactSubscriptionWithoutReleasingGenerationCharg status, ok := h.lastRouteStatus(waitingViewer.sessionID) return ok && status.State == "waiting" && status.Reason == "sfu-admission" }) - waitingBeforeRelease := h.countRouteStatus(waitingViewer.sessionID, "waiting") drainGate := make(chan struct{}) - h.roomControl.SetSubscriptionDrainBarrier(drainGate) + h.media.SetSubscriptionDrainBarrier(drainGate) h.locked(func() { h.doDisconnect(first) @@ -1881,22 +1778,17 @@ func TestRouterPhysicallyRemovesExactSubscriptionWithoutReleasingGenerationCharg }) h.waitFor("one diagnostic child", func() bool { return len(h.diagnostic(activeRoom.RoomID).Children) == 1 }) h.waitFor("the subscription drain attempt", func() bool { - return hasViewer(h.roomControl.SubscriptionDrainAttempts(), first.peerID) + return hasViewer(h.media.SubscriptionDrainAttempts(), first.peerID) }) if usage := h.usage(); usage != (sfu.Usage{Ingress: 2, Egress: 2}) { t.Fatalf("usage during drain = %+v", usage) } close(drainGate) h.waitFor("the drained subscription", func() bool { - return hasViewer(h.roomControl.DrainedSubscriptions(), first.peerID) + return hasViewer(h.media.DrainedSubscriptions(), first.peerID) }) - if usage := h.usage(); usage != (sfu.Usage{Ingress: 2, Egress: 2}) { - t.Fatalf("usage after drain = %+v", usage) - } - if count := h.countRouteStatus(waitingViewer.sessionID, "waiting"); count != waitingBeforeRelease { - t.Fatalf("waiting statuses = %d, want %d (a subscription drain must not wake waiters)", count, waitingBeforeRelease) - } + h.waitUsage(sfu.Usage{Ingress: 2, Egress: 1}) if err := h.close(); err != nil { t.Fatalf("close: %v", err) @@ -1980,151 +1872,6 @@ func TestRouterCreatesOneHostPublicationAndReusesExactViewerSubscriptions(t *tes } } -func TestRouterDiscardsSfuPreparationWhoseCursorBecomesStaleWhileAwaitingLiveKit(t *testing.T) { - h := newRouterHarness(t, routerHarnessOptions{capacity: 1, withSfu: true}) - created := h.createRoom() - host := h.connectHost(created, nil, "", "") - root := h.connectViewer(created, "prepare-stale-root") - h.locked(func() { - h.doComplete(host) - h.doComplete(root) - }) - direct := h.waitPreparedTransport(root.sessionID, "direct") - gate := make(chan struct{}) - waiting := h.connectViewer(created, "prepare-stale-waiting") - h.locked(func() { - h.doReady(root, int64(direct.Revision)) - h.roomControl.SetCreateBarrier(gate) - h.doRelay(root, 0) - h.doComplete(waiting) - }) - h.waitFor("the publication reservation", func() bool { return h.usage().Ingress == 1 }) - h.locked(func() { - h.doDisconnect(root) - close(gate) - }) - h.waitUsage(sfu.Usage{}) - if operation := h.diagnostic(created.RoomID).Operation; operation != nil { - t.Fatalf("operation = %+v, want none", operation) - } -} - -func TestRouterReschedulesRegularHostOfflineCheckAfterReadError(t *testing.T) { - h := newRouterHarness(t, routerHarnessOptions{capacity: 1, withSfu: true, sfuIngressCapacity: 2, hostOfflineCheckMs: 10}) - created := h.createRoom() - host, _, _ := h.establishSfuRoom(created) - release := make(chan struct{}) - t.Cleanup(func() { close(release) }) - h.roomControl.queueHostCheck( - func(context.Context, sfu.ResourceFence) (bool, error) { - return false, errors.New("temporary LiveKit read failure") - }, - func(context.Context, sfu.ResourceFence) (bool, error) { - <-release - return true, nil - }, - ) - h.disconnect(host) - - h.waitFor("two host checks", func() bool { - _, hostChecks, _ := h.roomControl.counts() - return hostChecks == 2 - }) - if usage := h.usage(); usage != (sfu.Usage{Ingress: 1, Egress: 2}) { - t.Fatalf("usage = %+v", usage) - } -} - -func TestRouterRechecksHostOfflineRetirementAfterLiveRouteOperation(t *testing.T) { - h := newRouterHarness(t, routerHarnessOptions{capacity: 1, withSfu: true, sfuIngressCapacity: 2, hostOfflineCheckMs: 10}) - created := h.createRoom() - host, first, second := h.establishSfuRoom(created) - var active activeViewerMediaEdge - h.locked(func() { - h.doRelay(second, 1) - active = h.doMustEdge(created.RoomID, first.peerID) - h.doFailed(first, routeFailedMessage(active.revision, "active", active.connectionID)) - }) - h.waitPreparedMatching(first.sessionID, "a direct rebuild", func(prepared protocol.RouteUpdatePrepareMessage) bool { - return int64(prepared.Revision) > active.revision && prepared.Candidate.Transport == "direct" - }) - retry := make(chan struct{}) - h.roomControl.queueHostCheck( - func(context.Context, sfu.ResourceFence) (bool, error) { return false, nil }, - func(context.Context, sfu.ResourceFence) (bool, error) { - <-retry - return false, nil - }, - ) - h.disconnect(host) - - h.waitFor("two host checks", func() bool { - _, hostChecks, _ := h.roomControl.counts() - return hostChecks == 2 - }) - if _, ok := h.activeEdge(created.RoomID, first.peerID); ok { - t.Fatal("the first viewer must have no active edge during the rebuild") - } - close(retry) - h.waitUsage(sfu.Usage{}) - if _, hostChecks, _ := h.roomControl.counts(); hostChecks != 2 { - t.Fatalf("host checks = %d, want 2", hostChecks) - } -} - -func TestRouterIgnoresOldHostOfflineResultAfterNewSfuGenerationCommits(t *testing.T) { - h := newRouterHarness(t, routerHarnessOptions{capacity: 1, withSfu: true, sfuIngressCapacity: 2, hostOfflineCheckMs: 10}) - created := h.createRoom() - host, first, second := h.establishSfuRoom(created) - oldRevision := max(h.mustActiveEdge(created.RoomID, first.peerID).revision, h.mustActiveEdge(created.RoomID, second.peerID).revision) - delayed := make(chan struct{}) - h.roomControl.queueHostCheck(func(context.Context, sfu.ResourceFence) (bool, error) { - <-delayed - return false, nil - }) - h.disconnect(host) - h.waitFor("the first host check", func() bool { - _, hostChecks, _ := h.roomControl.counts() - return hostChecks == 1 - }) - - replacementHost := h.connectHost(created, nil, "new-physical-host-client", "new-physical-host-session") - h.complete(replacementHost) - handled := map[string]struct{}{} - for index := 0; index < 3; index++ { - var next authenticatedRouteParticipant - var prepared protocol.RouteUpdatePrepareMessage - h.waitFor("a new-generation prepare", func() bool { - for _, viewer := range []authenticatedRouteParticipant{first, second} { - candidate, ok := h.preparedFor(viewer.sessionID) - if _, done := handled[candidate.Candidate.ConnectionID]; ok && int64(candidate.Revision) > oldRevision && !done { - next, prepared = viewer, candidate - return true - } - } - return false - }) - handled[prepared.Candidate.ConnectionID] = struct{}{} - h.routeReady(next, int64(prepared.Revision)) - } - h.waitUsage(sfu.Usage{Ingress: 1, Egress: 2}) - createdFences := h.roomControl.Created() - currentFence := createdFences[len(createdFences)-1] - if currentFence.PublicationGeneration == createdFences[0].PublicationGeneration { - t.Fatal("the replacement host must publish a new generation") - } - - close(delayed) - // TS: two microtask turns for the stale continuation to run. - time.Sleep(50 * time.Millisecond) - if usage := h.usage(); usage != (sfu.Usage{Ingress: 1, Egress: 2}) { - t.Fatalf("usage after the stale result = %+v", usage) - } - if slices.Contains(h.roomControl.Deleted(), currentFence) { - t.Fatal("the stale host-offline result must not retire the new generation") - } -} - func TestRouterResolvesExactDirectAndPeerRelayedViewerEvidenceSources(t *testing.T) { h := newRouterHarness(t, routerHarnessOptions{capacity: 1}) created := h.createRoom() @@ -2170,36 +1917,36 @@ func TestRouterResolvesExactDirectAndPeerRelayedViewerEvidenceSources(t *testing func TestRouterDrainsEveryPendingSfuPublicationDuringCloseEvenWhenOneDrainRejects(t *testing.T) { before := runtime.NumGoroutine() - h := newRouterHarness(t, routerHarnessOptions{capacity: 1, withSfu: true, prepareTimeoutMs: 1_000, sfuIngressCapacity: 2, drainRetryMs: 10_000}) + h := newRouterHarness(t, routerHarnessOptions{capacity: 1, withSfu: true, prepareTimeoutMs: 1_000, sfuIngressCapacity: 2_000}) roomA := h.createRoom() h.establishSfuRoom(roomA) roomB := h.createRoom() h.establishSfuRoom(roomB) - h.roomControl.SetFailDelete(true) + h.media.SetFailDelete(true) h.locked(func() { h.router.stopRoom(roomA.RoomID) h.router.stopRoom(roomB.RoomID) }) h.waitFor("two delete attempts", func() bool { - deletes, _, _ := h.roomControl.counts() + deletes, _, _ := h.media.counts() return deletes == 2 }) - h.roomControl.SetFailDelete(false) - fake := h.roomControl.FakeRoomControl - h.roomControl.setDeleteRoom(func(ctx context.Context, fence sfu.ResourceFence) error { + h.media.SetFailDelete(false) + fake := h.media.FakeMedia + h.media.setClosePublication(func(fence sfu.ResourceFence) error { if fence.RoomID == roomA.RoomID { return errors.New("room deletion failed") } - return fake.DeleteRoom(ctx, fence) + return fake.ClosePublication(fence) }) err := h.close() - if err == nil || !strings.Contains(err.Error(), "LiveKit drain failed during shutdown") { + if err == nil || !strings.Contains(err.Error(), "SFU drain failed during shutdown") { t.Fatalf("close error = %v, want the shutdown drain failure", err) } var deleted []string - for _, fence := range h.roomControl.Deleted() { + for _, fence := range h.media.Deleted() { deleted = append(deleted, fence.RoomID) } if !slices.Equal(deleted, []string{roomB.RoomID}) { @@ -2208,79 +1955,6 @@ func TestRouterDrainsEveryPendingSfuPublicationDuringCloseEvenWhenOneDrainReject expectGoroutinesSettled(t, before) } -func TestRouterRetriesFailedPhysicalDrainBeforeReusingExactSfuIdentity(t *testing.T) { - h := newRouterHarness(t, routerHarnessOptions{capacity: 1, withSfu: true, prepareTimeoutMs: 1_000, sfuIngressCapacity: 2, drainRetryMs: 10}) - created := h.createRoom() - _, first, _ := h.establishSfuRoom(created) - h.waitFor("the first viewer on the SFU", func() bool { - edge, ok := h.activeEdge(created.RoomID, first.peerID) - return ok && edge.upstream == protocol.SfuUpstream() - }) - - child := h.connectViewer(created, "sfu-peer-child") - h.locked(func() { - h.doRelay(first, 1) - h.doComplete(child) - }) - h.waitPreparedTransport(child.sessionID, "direct") - // The direct boundary of a 1000 ms operation is 500 ms of real time. - time.Sleep(510 * time.Millisecond) - sfuPrepare := h.waitPreparedTransport(child.sessionID, "sfu") - h.routeReady(child, int64(sfuPrepare.Revision)) - h.waitFor("the child on the SFU", func() bool { - edge, ok := h.activeEdge(created.RoomID, child.peerID) - return ok && edge.upstream == protocol.SfuUpstream() - }) - directPrepare := h.waitPreparedTransport(child.sessionID, "direct") - drainGate := make(chan struct{}) - h.locked(func() { - h.roomControl.queueDrain(func(context.Context, sfu.SubscriptionFence) error { - <-drainGate - return errors.New("transient participant drain failure") - }) - h.doReady(child, int64(directPrepare.Revision)) - }) - h.waitFor("the child on the first viewer", func() bool { - edge, ok := h.activeEdge(created.RoomID, child.peerID) - return ok && edge.upstream == protocol.PeerUpstream(first.peerID) - }) - h.waitFor("the child subscription drain attempt", func() bool { - return hasViewer(h.roomControl.drainAttempts(), child.peerID) - }) - if usage := h.usage(); usage != (sfu.Usage{Ingress: 1, Egress: 3}) { - t.Fatalf("usage = %+v", usage) - } - - h.routeFailed(child, routeFailedMessage(int64(directPrepare.Revision), "active", directPrepare.Candidate.ConnectionID)) - h.waitFor("a rebuild operation", func() bool { return h.diagnostic(created.RoomID).Operation != nil }) - if prepared, _ := h.preparedFor(child.sessionID); prepared.Revision != directPrepare.Revision { - t.Fatalf("prepare revision during the pending drain = %d, want %d", prepared.Revision, directPrepare.Revision) - } - - close(drainGate) - h.waitFor("the drain retry", func() bool { - _, _, drains := h.roomControl.counts() - return drains == 2 - }) - reusedSfu := h.waitPreparedMatching(child.sessionID, "the reused SFU prepare", func(prepared protocol.RouteUpdatePrepareMessage) bool { - return int64(prepared.Revision) > int64(directPrepare.Revision) && prepared.Candidate.Transport == "sfu" - }) - h.routeReady(child, int64(reusedSfu.Revision)) - h.waitFor("the child back on the SFU", func() bool { - edge, ok := h.activeEdge(created.RoomID, child.peerID) - return ok && edge.upstream == protocol.SfuUpstream() - }) - if !hasViewer(h.roomControl.DrainedSubscriptions(), child.peerID) { - t.Fatal("the retried drain must have removed the child") - } - if usage := h.usage(); usage != (sfu.Usage{Ingress: 1, Egress: 3}) { - t.Fatalf("usage after reuse = %+v", usage) - } - if edge := h.mustActiveEdge(created.RoomID, first.peerID); edge.upstream != protocol.SfuUpstream() { - t.Fatalf("first viewer upstream = %v", edge.upstream) - } -} - func TestRouterSerializesFreshSfuConfigurationWithoutMakingRefreshOneShot(t *testing.T) { h := newRouterHarness(t, routerHarnessOptions{capacity: 1, withSfu: true}) created := h.createRoom() @@ -2301,55 +1975,10 @@ func TestRouterSerializesFreshSfuConfigurationWithoutMakingRefreshOneShot(t *tes h.waitFor("the second refresh", func() bool { return configCount() == initialCount+2 }) } -func TestRouterFailsExactActiveSfuRouteWhenFreshConfigurationCannotBeIssued(t *testing.T) { - h := newRouterHarness(t, routerHarnessOptions{capacity: 1, withSfu: true}) - created := h.createRoom() - _, first, _ := h.establishSfuRoom(created) - var active activeViewerMediaEdge - h.locked(func() { - active = h.doMustEdge(created.RoomID, first.peerID) - if active.upstream != protocol.SfuUpstream() { - t.Fatalf("upstream = %v", active.upstream) - } - h.failNextTokenIssue() - h.router.refreshSfu(first, active.revision) - }) - h.waitFor("the active SFU route to fail", func() bool { - edge, ok := h.activeEdge(created.RoomID, first.peerID) - return !ok || edge != active - }) -} - -// hybrid-media-router.ts:1129-1132: a throw from roomControl or tokenIssuer +// hybrid-media-router.ts:1129-1132: a throw from media or tokenIssuer // rejected the pump promise and its .catch told the Host. In Go the throw is a // panic raised while mu is released for the token issue, so the pump must come // back holding mu; otherwise drivePump's deferred Unlock aborts the process. -func TestRouterTellsHostWhenCandidatePreparationPanicsWithTheLockReleased(t *testing.T) { - h := newRouterHarness(t, routerHarnessOptions{capacity: 1, withSfu: true}) - created := h.createRoom() - host, _, _ := h.establishSfuRoom(created) - joining := h.connectViewer(created, "sfu-panic") - - h.locked(func() { - h.panicNextTokenIssue() - h.doComplete(joining) - }) - - h.waitFor("the host to be told that media routing failed", func() bool { - for _, message := range h.messages(host.sessionID) { - failure, ok := message.(protocol.ErrorMessage) - if ok && failure.Code == "SERVER_ERROR" && failure.Message == "Media routing failed" { - return true - } - } - return false - }) - // The router still owns its lock: it settles and serves the next call. - h.settle() - if _, ok := h.activeEdge(created.RoomID, joining.peerID); ok { - t.Fatal("the failed candidate must not have been committed") - } -} func TestRouterEmitsTypedExhaustionAndClearsDiagnosticsOnDepartureAndRoomDeletion(t *testing.T) { h := newRouterHarness(t, routerHarnessOptions{capacity: 1}) @@ -2437,67 +2066,10 @@ func TestRouterEmitsTypedExhaustionWhenReconciliationHasNoCandidate(t *testing.T // the per-room-name FIFO keeps LiveKit calls in decision order. // --------------------------------------------------------------------------- -func TestRouterTurnsRunInDecisionOrderPerRoomName(t *testing.T) { - var mu sync.Mutex - r := &router{mu: &mu, roomTurns: map[string]chan struct{}{}} - mu.Lock() - first := r.takeTurn("room") - second := r.takeTurn("room") - other := r.takeTurn("other") - mu.Unlock() - - var order []string - var orderMu sync.Mutex - record := func(name string) { - orderMu.Lock() - defer orderMu.Unlock() - order = append(order, name) - } - release := make(chan struct{}) - done := make(chan struct{}) - go func() { - second.run(func() { record("second") }) - done <- struct{}{} - }() - go func() { - other.run(func() { record("other") }) - done <- struct{}{} - }() - <-done // "other" is independent and finishes first - go func() { - first.run(func() { - <-release - record("first") - }) - done <- struct{}{} - }() - time.Sleep(10 * time.Millisecond) - orderMu.Lock() - if !slices.Equal(order, []string{"other"}) { - t.Fatalf("second must wait for first: %v", order) - } - orderMu.Unlock() - close(release) - <-done - <-done - orderMu.Lock() - defer orderMu.Unlock() - if !slices.Equal(order, []string{"other", "first", "second"}) { - t.Fatalf("order = %v", order) - } - mu.Lock() - r.dropTurn(first) - if _, still := r.roomTurns["room"]; !still { - t.Fatal("dropping a superseded turn must keep the tail") - } - r.dropTurn(second) - if _, still := r.roomTurns["room"]; still { - t.Fatal("dropping the tail must forget the room name") - } - mu.Unlock() -} - func TestRouterDebugFlagParsesLikeNodeDebug(t *testing.T) { + previous := slog.Default() + t.Cleanup(func() { slog.SetDefault(previous) }) + slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, nil))) cases := map[string]bool{"": false, "route": true, "http,route": true, " route ": true, "router": false} keys := make([]string, 0, len(cases)) for key := range cases { @@ -2505,8 +2077,17 @@ func TestRouterDebugFlagParsesLikeNodeDebug(t *testing.T) { } sort.Strings(keys) for _, key := range keys { + t.Setenv("SCREENER_DEBUG", key) if got := routeDebugFlag(key); got != cases[key] { t.Fatalf("routeDebugFlag(%q) = %v, want %v", key, got, cases[key]) } + if got := routeDebugEnabled(); got != cases[key] { + t.Fatalf("routeDebugEnabled with %q = %v, want %v", key, got, cases[key]) + } + } + t.Setenv("SCREENER_DEBUG", "") + slog.SetDefault(slog.New(slog.NewTextHandler(io.Discard, &slog.HandlerOptions{Level: slog.LevelDebug}))) + if !routeDebugEnabled() { + t.Fatal("the active diagnostic logger did not enable route events") } } diff --git a/internal/server/signal/server.go b/internal/server/signal/server.go index 643ebf3..4a51740 100644 --- a/internal/server/signal/server.go +++ b/internal/server/signal/server.go @@ -10,7 +10,7 @@ // Server.mu is the one lock of the effect layer: it guards every field of // the server and the router, the room.Store and the route controllers. Every // handler, timer callback and goroutine takes it; it is released only around -// I/O (the password KDF, LiveKit, token issue), after which exactly the guard +// I/O (the password KDF and media transport), after which exactly the guard // the TypeScript ran after its await is re-run. Router methods and hooks are // called with mu held and never lock. package signal @@ -61,16 +61,12 @@ const ( defaultCleanupIntervalMs = 30_000 ) -// SfuFallback is SfuFallbackOptions: the LiveKit fallback the media router +// SfuFallback is the embedded media runtime the media router // may use. A zero timeout means the TypeScript default. type SfuFallback struct { - URL string - TokenIssuer sfu.TokenIssuer - Admission *sfu.Admission - RoomControl sfu.RoomControl - PrepareTimeoutMs int64 - DrainRetryMs int64 - HostOfflineCheckMs int64 + Media sfu.Runtime + Admission *sfu.Admission + PrepareTimeoutMs int64 } // Options is SignalingOptions. Every zero value is the TypeScript `undefined` @@ -78,7 +74,7 @@ type SfuFallback struct { type Options struct { Store *room.Store EndpointMediaCopyCapacity int - // SfuFallback is nil when the LiveKit fallback is disabled. + // SfuFallback is nil when embedded SFU media is disabled. SfuFallback *SfuFallback // Ice is the derived wire ICE configuration (config.IceConfig). Ice protocol.IceConfig @@ -225,6 +221,9 @@ func New(options Options) (*Server, error) { if options.Store == nil { return nil, errors.New("signaling requires a room store") } + if options.SfuFallback != nil && (options.SfuFallback.Media == nil || options.SfuFallback.Admission == nil) { + return nil, errors.New("SFU fallback requires media and resource admission") + } s := &Server{ store: options.Store, endpointMediaCopyCapacity: options.EndpointMediaCopyCapacity, @@ -272,13 +271,9 @@ func New(options Options) (*Server, error) { var fallback *sfuFallback if options.SfuFallback != nil { fallback = &sfuFallback{ - url: options.SfuFallback.URL, - tokenIssuer: options.SfuFallback.TokenIssuer, - admission: options.SfuFallback.Admission, - roomControl: options.SfuFallback.RoomControl, - prepareTimeoutMs: options.SfuFallback.PrepareTimeoutMs, - drainRetryMs: options.SfuFallback.DrainRetryMs, - hostOfflineCheckMs: options.SfuFallback.HostOfflineCheckMs, + media: options.SfuFallback.Media, + admission: options.SfuFallback.Admission, + prepareTimeoutMs: options.SfuFallback.PrepareTimeoutMs, } } mediaRouter, err := buildRouter(routerOptions{ @@ -305,8 +300,7 @@ func New(options Options) (*Server, error) { return s, nil } -// buildRouter turns the HybridMediaRouter constructor's throw (an invalid -// SFU URL) into the error New returns. +// buildRouter returns constructor assertions through the startup error path. func buildRouter(options routerOptions) (r *router, err error) { defer func() { if recovered := recover(); recovered != nil { @@ -481,10 +475,10 @@ func (s *Server) Close(ctx context.Context) error { grace := time.NewTimer(serviceRestartCloseGrace) defer grace.Stop() // TS close() ran `this.closing = true` and the router's synchronous close - // prefix (its own closing flag, every deadline / host-offline / retry + // prefix (its own closing flag, every deadline and operation // timer, the drains) in one tick; only the drain waits yielded. Keeping mu // held across that prefix reproduces it: a router timer callback that - // fires now cannot slip in between and start a pump against LiveKit. + // fires now cannot slip in between and start a new media operation. routeCloseError := s.router.close(ctx) s.mu.Unlock() @@ -977,11 +971,6 @@ func (s *Server) completeAuthentication(sess *session, request authRequest, part share := s.shares[roomID] routePolicy := s.routePolicyOf(roomID) - var sfuStandbyURL *string - if s.sfuFallback != nil && share.routePolicy != nil && !routePolicy.PeerOnly { - standby := s.sfuFallback.URL - sfuStandbyURL = &standby - } qualitySettings := protocol.DefaultQualitySettings if share.qualitySettings != nil { qualitySettings = *share.qualitySettings @@ -1010,7 +999,6 @@ func (s *Server) completeAuthentication(sess *session, request authRequest, part RouteRevision: protocol.Int(hybridState.routeRevision), RouteAssignment: hybridState.routeAssignment, QualitySettings: qualitySettings, - SfuStandbyURL: sfuStandbyURL, Role: protocol.RoleHost, ViewerPasswordEnabled: participant.ViewerPasswordEnabled, }) @@ -1036,7 +1024,6 @@ func (s *Server) completeAuthentication(sess *session, request authRequest, part RouteRevision: protocol.Int(hybridState.routeRevision), RouteAssignment: hybridState.routeAssignment, QualitySettings: qualitySettings, - SfuStandbyURL: sfuStandbyURL, Role: protocol.RoleViewer, }) } @@ -1120,10 +1107,6 @@ func (s *Server) settleHostShare(roomID, sessionID string, request authRequest) RoutePolicy: routePolicy, } // Hazard 3: no `has(roomId)` term here, unlike `authenticated`. - if s.sfuFallback != nil && !routePolicy.PeerOnly { - standby := s.sfuFallback.URL - message.SfuStandbyURL = &standby - } s.sendToSession(viewer.SessionID, message) } } @@ -1250,6 +1233,8 @@ func (s *Server) handleAuthenticatedMessage(sess *session, authenticated *authen s.router.handleRouteMediaUnavailable(routeParticipant(sess, authenticated), m) case protocol.RouteFailedMessage: s.router.handleRouteFailed(routeParticipant(sess, authenticated), m) + case protocol.SfuSignalMessage: + s.router.handleSfuSignal(routeParticipant(sess, authenticated), m) case protocol.RefreshSfuMessage: if s.sfuFallback == nil { s.sendError(sess, "FORBIDDEN", "SFU fallback is not enabled") @@ -1333,7 +1318,7 @@ func (s *Server) handleAuthenticatedMessage(sess *session, authenticated *authen return } if err := s.abandonRoom(authenticated.roomID); err != nil { - s.logger.Error("Room abandonment failed", "error", err) + s.logger.Error("Room abandonment failed", "errorType", fmt.Sprintf("%T", err)) s.sendError(sess, "SERVER_ERROR", "Room could not be abandoned") } } diff --git a/internal/server/signal/server_test.go b/internal/server/signal/server_test.go index f3d91a5..f9ae1a6 100644 --- a/internal/server/signal/server_test.go +++ b/internal/server/signal/server_test.go @@ -71,6 +71,48 @@ func TestCreateRoomReleasesLockWhenDerivationOrCommitPanics(t *testing.T) { } } +func TestIncomingStorageFailureReleasesLockWithoutSuppressingPanic(t *testing.T) { + database, err := room.NewDatabase(filepath.Join(t.TempDir(), "rooms.sqlite")) + if err != nil { + t.Fatal(err) + } + store := newStore(t, 1, nil, database) + defer store.Close() + created := createStoreRoom(t, store, protocol.CodeEntryOpen, "") + sess := newSession(nil, true) + defer sess.cancel() + participant, err := store.ConnectParticipant(room.ConnectParticipantInput{ + RoomID: created.RoomID, Role: protocol.RoleHost, Token: created.HostToken, + ClientID: "failed-storage-host", SessionID: sess.sessionID, + }) + if err != nil { + t.Fatal(err) + } + const generation = "failed_storage_share_generation" + sess.authenticated = &authenticatedSession{roomID: created.RoomID, role: protocol.RoleHost, + peerID: participant.PeerID, shareGeneration: generation} + server := &Server{store: store, shares: map[string]roomShare{created.RoomID: {generation: generation}}} + server.sessions.Set(sess, struct{}{}) + if err = database.Close(); err != nil { + t.Fatal(err) + } + func() { + defer func() { + failure, ok := recover().(error) + if !ok || !strings.Contains(failure.Error(), "room store write failed while stopping sharing") { + t.Errorf("handler did not preserve its storage panic: %v", failure) + } + }() + server.handleIncoming(sess, websocket.MessageText, + []byte(`{"type":"stop-sharing","shareGeneration":"`+generation+`"}`)) + }() + if !server.mu.TryLock() { + server.mu.Unlock() // Allow test cleanup if the regression returns. + t.Fatal("storage panic left the message lock held before reader cleanup") + } + server.mu.Unlock() +} + // --------------------------------------------------------------------------- // harness // --------------------------------------------------------------------------- @@ -2713,10 +2755,8 @@ func TestSignalCloseMarksRouterClosingBeforeReleasingTheLock(t *testing.T) { afterFunc: timers.afterFunc, now: now.now, sfu: &SfuFallback{ - URL: "wss://sfu.example.test", - TokenIssuer: &fakeTokenIssuer{}, Admission: sfu.NewAdmission(sfu.AdmissionOptions{IngressCapacity: 2, EgressCapacity: 20}), - RoomControl: control, + Media: control, PrepareTimeoutMs: 300, }, }) diff --git a/internal/server/signal/session.go b/internal/server/signal/session.go index db33c52..0dfd7fe 100644 --- a/internal/server/signal/session.go +++ b/internal/server/signal/session.go @@ -176,25 +176,29 @@ func (s *Server) armAuthenticationTimer(sess *session) { // the connection fails and then runs handleDisconnect exactly once (D6). func (s *Server) readLoop(sess *session) { defer func() { + defer close(sess.readerDone) sess.cancel() s.mu.Lock() + defer s.mu.Unlock() s.handleDisconnect(sess) - s.mu.Unlock() - close(sess.readerDone) }() for { kind, data, err := sess.conn.Read(sess.ctx) if err != nil { return } - s.mu.Lock() - if kind == websocket.MessageBinary { - // TS: if (isBinary) rejectInvalidMessage(socket) - s.rejectInvalidMessage(sess) - } else { - s.handleMessage(sess, data) - } - s.mu.Unlock() + s.handleIncoming(sess, kind, data) + } +} + +func (s *Server) handleIncoming(sess *session, kind websocket.MessageType, data []byte) { + s.mu.Lock() + defer s.mu.Unlock() + if kind == websocket.MessageBinary { + // TS: if (isBinary) rejectInvalidMessage(socket) + s.rejectInvalidMessage(sess) + } else { + s.handleMessage(sess, data) } } diff --git a/internal/server/stun/server.go b/internal/server/stun/server.go new file mode 100644 index 0000000..8dc21aa --- /dev/null +++ b/internal/server/stun/server.go @@ -0,0 +1,54 @@ +// Package stun owns the UDP listeners for in-process STUN discovery. +package stun + +import ( + "context" + "errors" + "io" + "net" + + "github.com/pion/logging" + "github.com/pion/turn/v5" +) + +type Server struct { + *turn.Server + Addresses []net.Addr +} + +// Listen binds the ordinary endpoint or the three-destination NAT survey. +// The caller closes the returned server with the application lifecycle. +func Listen(ctx context.Context, addresses []string) (*Server, error) { + if len(addresses) != 1 && len(addresses) != 3 { + return nil, errors.New("STUN requires one or three UDP listeners") + } + var listeners []turn.PacketConnConfig + var bound []net.Addr + closeListeners := func() { + for _, listener := range listeners { + _ = listener.PacketConn.Close() + } + } + for _, address := range addresses { + connection, err := (&net.ListenConfig{}).ListenPacket(ctx, "udp4", address) + if err != nil { + closeListeners() + return nil, err + } + listeners = append(listeners, turn.PacketConnConfig{PacketConn: connection}) + bound = append(bound, connection.LocalAddr()) + } + logger := logging.NewDefaultLoggerFactory() + // Library packet logs include remote addresses; do not export them. + logger.Writer = io.Discard + server, err := turn.NewServer(turn.ServerConfig{ + PacketConnConfigs: listeners, + LoggerFactory: logger, + // No AuthHandler or relay generator: Pion's STUN-only mode. + }) + if err != nil { + closeListeners() + return nil, err + } + return &Server{Server: server, Addresses: bound}, nil +} diff --git a/internal/server/stun/server_test.go b/internal/server/stun/server_test.go new file mode 100644 index 0000000..958f644 --- /dev/null +++ b/internal/server/stun/server_test.go @@ -0,0 +1,99 @@ +package stun + +import ( + "bytes" + "net" + "testing" + "time" + + "github.com/pion/stun/v3" +) + +func TestThreeSTUNListenersWithoutRelayAndWithCompleteShutdown(t *testing.T) { + server, err := Listen(t.Context(), []string{"127.0.0.1:0", "127.0.0.1:0", "127.0.0.1:0"}) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = server.Close() }) + client, err := net.ListenPacket("udp4", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer client.Close() + for _, address := range server.Addresses { + response := exchange(t, client, address, stun.BindingRequest) + var mapped stun.XORMappedAddress + if response.Type != stun.BindingSuccess || mapped.GetFrom(response) != nil { + t.Fatal("STUN did not return a binding response") + } + local := client.LocalAddr().(*net.UDPAddr) + if mapped.Port != local.Port || !mapped.IP.Equal(local.IP) { + t.Fatal("STUN returned a different socket's address") + } + response = exchange(t, client, address, stun.NewType(stun.MethodAllocate, stun.ClassRequest)) + if response.Type.Class != stun.ClassErrorResponse || server.AllocationCount() != 0 { + t.Fatal("STUN-only listener allowed TURN allocation") + } + } + if err := server.Close(); err != nil { + t.Fatal(err) + } + for _, address := range server.Addresses { + reopened, err := net.ListenPacket("udp4", address.String()) + if err != nil { + t.Fatal("STUN socket remained open after shutdown") + } + _ = reopened.Close() + } +} + +func TestOccupiedListenerRollsBackOnlyNewSockets(t *testing.T) { + first, err := net.ListenPacket("udp4", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + firstAddress := first.LocalAddr().String() + _ = first.Close() + occupied, err := net.ListenPacket("udp4", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer occupied.Close() + server, err := Listen(t.Context(), []string{firstAddress, occupied.LocalAddr().String(), "127.0.0.1:0"}) + if err == nil || server != nil { + if server != nil { + _ = server.Close() + } + t.Fatal("occupied listener was accepted") + } + reopened, err := net.ListenPacket("udp4", firstAddress) + if err != nil { + t.Fatal("failed startup retained an earlier socket") + } + _ = reopened.Close() + if err := occupied.SetDeadline(time.Now().Add(time.Second)); err != nil { + t.Fatal("failed startup closed a foreign socket") + } +} + +func exchange(t *testing.T, client net.PacketConn, destination net.Addr, kind stun.MessageType) *stun.Message { + t.Helper() + request := stun.MustBuild(stun.TransactionID, kind) + if err := client.SetDeadline(time.Now().Add(time.Second)); err != nil { + t.Fatal(err) + } + if _, err := client.WriteTo(request.Raw, destination); err != nil { + t.Fatal(err) + } + buffer := make([]byte, 1500) + count, source, err := client.ReadFrom(buffer) + if err != nil { + t.Fatal(err) + } + response := &stun.Message{Raw: buffer[:count]} + if response.Decode() != nil || source.String() != destination.String() || + !bytes.Equal(response.TransactionID[:], request.TransactionID[:]) { + t.Fatal("STUN response identity did not match") + } + return response +} diff --git a/licenses/README.md b/licenses/README.md index 9d19dac..dcfdb62 100644 --- a/licenses/README.md +++ b/licenses/README.md @@ -4,14 +4,16 @@ Screener-owned code uses the root [MIT license](../LICENSE). Dependencies retain their own licenses; MIT does not replace their copyright notices or terms. The [pinned source index](./upstream.json) records exact upstream commits and -SHA-256 values for Cloudflared, libvpx, and npm packages that omit license +SHA-256 values for Cloudflared, the WebRTC SDK, and npm packages that omit license files. The retained text normalizes line endings and incidental whitespace only. The Cloudflared text includes its source distribution's root and all vendored LICENSE, COPYING, and NOTICE files, including platform variants. The protobuf text also preserves Google's BSD-licensed varint attribution. -The Windows capture adapter statically links libvpx; its copyright, patent grant -and author list are included in the Client's native notices. +The Windows capture adapter statically links the pinned WebRTC SDK, including +its libvpx encoder. The Client's native notices retain the SDK's collected +third-party license texts; these dependencies do not become MIT-licensed merely +because Screener's adapter is MIT-licensed. Linux packages that include the native capture binary include `linux-system-dependencies.txt`, the full LGPL-3.0-only text for libportal, and the full GPL-3.0 text referenced by that license. The system shared libraries @@ -26,6 +28,18 @@ and the license files of the modules actually compiled into that binary; the Client package also carries the pinned Cloudflared notices. Missing license texts or changed pinned contents stop packaging. +LiveKit media-core packages are referenced as an unmodified, version-pinned Go +module under Apache-2.0. Original Screener adapters remain MIT; adapted upstream +probe orchestration retains its Apache-2.0 attribution in the source. Redistribution retains +the module's license and applicable notices through the same Go notice collector. +Any later copied or modified upstream file must retain its attribution and mark +the modification. Prefer public API composition over a maintained source fork. + +Dependency updates are explicit rebuilds, not live updates in a running binary. +Review relevant upstream advisories and actual reachable dependencies at release; +an affected serious vulnerability warrants an earlier update. Mature media code +is not exempt from vulnerability checks or focused upgrade verification. + The Linux license texts come from the GNU license pages: `https://www.gnu.org/licenses/lgpl-3.0.txt` and `https://www.gnu.org/licenses/gpl-3.0.txt`. diff --git a/licenses/libvpx-1.17.0.txt b/licenses/libvpx-1.17.0.txt deleted file mode 100644 index 3e98776..0000000 --- a/licenses/libvpx-1.17.0.txt +++ /dev/null @@ -1,327 +0,0 @@ -libvpx 1.17.0 -Source: https://github.com/webmproject/libvpx/tree/6df3ec34557879fff673706f4a1d9fbd0f3a6f0e - -===== LICENSE ===== - -Copyright (c) 2010, The WebM Project authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright - notice, this list of conditions and the following disclaimer. - - * Redistributions in binary form must reproduce the above copyright - notice, this list of conditions and the following disclaimer in - the documentation and/or other materials provided with the - distribution. - - * Neither the name of Google, nor the WebM Project, nor the names - of its contributors may be used to endorse or promote products - derived from this software without specific prior written - permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. - -===== PATENTS ===== - -Additional IP Rights Grant (Patents) ------------------------------------- - -"These implementations" means the copyrightable works that implement the WebM -codecs distributed by Google as part of the WebM Project. - -Google hereby grants to you a perpetual, worldwide, non-exclusive, no-charge, -royalty-free, irrevocable (except as stated in this section) patent license to -make, have made, use, offer to sell, sell, import, transfer, and otherwise -run, modify and propagate the contents of these implementations of WebM, where -such license applies only to those patent claims, both currently owned by -Google and acquired in the future, licensable by Google that are necessarily -infringed by these implementations of WebM. This grant does not include claims -that would be infringed only as a consequence of further modification of these -implementations. If you or your agent or exclusive licensee institute or order -or agree to the institution of patent litigation or any other patent -enforcement activity against any entity (including a cross-claim or -counterclaim in a lawsuit) alleging that any of these implementations of WebM -or any code incorporated within any of these implementations of WebM -constitute direct or contributory patent infringement, or inducement of -patent infringement, then any patent rights granted to you under this License -for these implementations of WebM shall terminate as of the date such -litigation is filed. - -===== AUTHORS ===== - -# This file is automatically generated from the git commit history -# by tools/gen_authors.sh. - -Aaron Watry -Abo Talib Mahfoodh -Adam B. Goode -Adrian Grange -Ahmad Sharif -Aidan Welch -Aleksey Vasenev -Alexander Potapenko -Alexander Voronov -Alexandra Hájková -Aℓex Converse -Alex Davicenko -Alexis Ballier -Alok Ahuja -Alpha Lam -A.Mahfoodh -Ami Fischman -Andoni Morales Alastruey -Andres Calderon Jaramillo -Andres Mejia -Andrew Lewis -Andrew Russell -Andrew Salkeld -Andrey Filipenkov -Angie Chen -Angie Chiang -Anton Venema -Anupam Pandey -Aron Rosenberg -Attila Nagy -Birk Magnussen -Bohan Li -Brian Foley -Brion Vibber -Casey Smalley -changjun.yang -Charles 'Buck' Krasic -Cheng Chen -Chen Wang -Cherma Rajan A -Chi Yo Tsai -chm -Chris Cunningham -Christian Duvivier -Chunbo Hua -Chun-Min Chang -Clement Courbet -CodeByMoriarty -Daniel Cheng -Daniele Castagna -Daniel Kang -Daniel Sommermann -Dan Zhu -Darryl Pogue -David Benjamin -Deb Mukherjee -Deepa K G -Diksha Singh -Dim Temp -Dmitry Kovalev -Dragan Mrdjan -Ed Baker -Ehsan Akhgari -Elliott Karpilovsky -Erik Niemeyer -Fabio Pedretti -Feng, Feifei -Florian Mayer -Frank Galligan -Fredrik Söderquist -Fritz Koenig -Fyodor Kyslov -Gabriel Marin -Gaute Strokkenes -George Steed -Gerda Zsejke More -Geza Lore -Ghislain MARY -Giuseppe Scrivano -Gordana Cmiljanovic -Gregor Jasny -Guillaume Martres -Guillermo Ballester Valor -Hang Nguyen -Hangyu Kuang -Hanno Böck -Han Shen -Hao Chen -Hari Limaye -Harish Mahendrakar -Henrik Lundin -Hien Ho -Hirokazu Honda -Hui Su -Ilya Kurdyukov -Ivan Krasin -Ivan Maltz -Jacek Caban -Jacky Chen -James Berry -James Touton -James Yu -James Zern -Jan Gerber -Jan-Ivar Bruaroey -Jan Kratochvil -Janne Salonen -Jean-Yves Avenard -Jeff Faust -Jeff Muizelaar -Jeff Petkau -Jeremy Dorfman -Jeremy Leconte -Jerome Jiang -Jia Jia -Jianhui Dai -Jian Zhou -Jim Bankoski -jinbo -Jin Bo -Jingning Han -Joel Fernandes -Joey Parrish -Johann -Johann Koenig -John Koleszar -Johnny Klonaris -John Stark -Jonathan Wright -Jon Kunkee -Jorge E. Moreira -Joshua Bleecher Snyder -Joshua Litt -Julia Robson -Justin Clift -Justin Lebar -Kaustubh Raste -Kexy Biscuit -KO Myung-Hun -Konstantinos Margaritis -Kyle Siefring -Lawrence Velázquez -L. E. Segovia -Linfeng Zhang -Lin Zheng -Liu Peng -Li Zhang -Lou Quillio -Luca Barbato -Luc Trudeau -Lu Wang -Makoto Kato -Mans Rullgard -Marco Paniconi -Mark Mentovai -Martin Ettl -Martin Storsjö -Matthew Heaney -Matthias Räncker -Michael Horowitz -Michael Kohler -Michał Janiszewski -Mike Frysinger -Mike Hommey -Mikhal Shemer -Mikko Koivisto -Min Chen -Minghai Shang -Min Ye -Mirko Bonadei -Moriyoshi Koizumi -Morton Jonuschat -Nathan E. Egge -Neeraj Gadgil -Neil Birkbeck -Nico Weber -Niveditha Rau -Parag Salasakar -Pascal Massimino -Patrik Westin -Paul Wilkins -Pavol Rusnak -Paweł Hajdan -Pengchong Jin -Peter Boström -Peter Collingbourne -Peter de Rivaz -Peter Kasting -Philip Jägenstedt -Philippe Antoine -Philipp Hancke -Priit Laes -Rafael Ávila de Espíndola -Rafaël Carré -Rafael de Lucena Valle -Rahul Chaudhry -Ralph Giles -Ram Mohan M -Ranjit Kumar Tulabandu -Raphael Kubo da Costa -Ravi Chaudhary -Ritu Baldwa -Rob Bradford -Ronald S. Bultje -Rui Ueyama -Sai Deng -Salome Thirot -Sami Pietilä -Sam James -Sarah Parker -Sasi Inguva -Scott Graham -Scott LaVarnway -Sean McGovern -Sergey Kolomenkin -Sergey Silkin -Sergey Ulanov -Shimon Doodkin -Shiyou Yin -Shubham Tandle -Shunyao Li -Sreerenj Balachandran -Stefan Holmer -Suman Sunkara -Supradeep T R -Sylvestre Ledru -Taekhyun Kim -Takanori MATSUURA -Tamar Levy -Tao Bai -Tero Rintaluoma -Thijs Vermeir -Tim Kopp -Timothy B. Terriberry -Tom Finegan -Tristan Matthews -Urvang Joshi -Venkatarama NG. Avadhani -Vignesh Venkatasubramanian -Vitaly Buka -Vlad Tsyrklevich -Wan-Teh Chang -Wonkap Jang -Xiahong Bao -Xiwei Gu -Yaowu Xu -Yi Luo -Yongseok Jeon -Yongzhe Wang -yuanhecai -Yue Chen -Yun Liu -Yunqing Wang -Yury Gitman -Zoe Liu -Zoltan Kuscsik -zuxy -Google Inc. -The Mozilla Foundation -The Xiph.Org Foundation diff --git a/licenses/upstream.json b/licenses/upstream.json index ee1af1a..8613514 100644 --- a/licenses/upstream.json +++ b/licenses/upstream.json @@ -1,8 +1,8 @@ { - "libvpx@1.17.0": { - "file": "libvpx-1.17.0.txt", - "source": "https://github.com/webmproject/libvpx/tree/6df3ec34557879fff673706f4a1d9fbd0f3a6f0e", - "sha256": "3042109b4fb2a8d44e52c6241eaa8a1a9b4bfdbd632e67b7d91b4d1db36d653b" + "webrtc@m152.7977.0.2": { + "file": "webrtc-m152.7977.0.2.txt", + "source": "https://github.com/shiguredo-webrtc-build/webrtc-build/releases/tag/m152.7977.0.2", + "sha256": "bf8b75cf620f754c60fc5efa8eb20ed543540aca1b91c63e6ac78f86b85844b0" }, "@bufbuild/protobuf@1.10.1": { "file": "bufbuild-protobuf-1.10.1.txt", diff --git a/licenses/webrtc-m152.7977.0.2.txt b/licenses/webrtc-m152.7977.0.2.txt new file mode 100644 index 0000000..30914e0 --- /dev/null +++ b/licenses/webrtc-m152.7977.0.2.txt @@ -0,0 +1,2090 @@ +# webrtc +``` +Copyright (c) 2011, The WebRTC project authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` + +# abseil-cpp +``` + + Apache License + Version 2.0, January 2004 + https://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +``` + +# boringssl +``` + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +Licenses for support code +------------------------- + +Parts of the TLS test suite are under the Go license. This code is not included +in BoringSSL (i.e. libcrypto and libssl) when compiled, however, so +distributing code linked against BoringSSL does not trigger this license: + +Copyright (c) 2009 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` + +# compiler-rt +``` +============================================================================== +The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: +============================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + +============================================================================== +Software from third parties included in the LLVM Project: +============================================================================== +The LLVM Project contains third party software which is under different license +terms. All such code will be identified clearly using at least one of two +mechanisms: +1) It will be in a separate directory tree with its own `LICENSE.txt` or + `LICENSE` file at the top containing the specific license and restrictions + which apply to that software, or +2) It will contain specific license and restriction terms at the top of every + file. + +============================================================================== +Legacy LLVM License (https://llvm.org/docs/DeveloperPolicy.html#legacy): +============================================================================== + +The compiler_rt library is dual licensed under both the University of Illinois +"BSD-Like" license and the MIT license. As a user of this code you may choose +to use it under either license. As a contributor, you agree to allow your code +to be used under both. + +Full text of the relevant licenses is included below. + +============================================================================== + +University of Illinois/NCSA +Open Source License + +Copyright (c) 2009-2019 by the contributors listed in CREDITS.TXT + +All rights reserved. + +Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +============================================================================== + +Copyright (c) 2009-2015 by the contributors listed in CREDITS.TXT + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +``` + +# dav1d +``` +Copyright © 2018, VideoLAN and dav1d authors +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +1. Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR +ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES +(INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND +ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` + +# fft +``` +/* + * Copyright(c)1995,97 Mark Olesen <olesen@me.QueensU.CA> + * Queen's Univ at Kingston (Canada) + * + * Permission to use, copy, modify, and distribute this software for + * any purpose without fee is hereby granted, provided that this + * entire notice is included in all copies of any software which is + * or includes a copy or modification of this software and in all + * copies of the supporting documentation for such software. + * + * THIS SOFTWARE IS BEING PROVIDED "AS IS", WITHOUT ANY EXPRESS OR + * IMPLIED WARRANTY. IN PARTICULAR, NEITHER THE AUTHOR NOR QUEEN'S + * UNIVERSITY AT KINGSTON MAKES ANY REPRESENTATION OR WARRANTY OF ANY + * KIND CONCERNING THE MERCHANTABILITY OF THIS SOFTWARE OR ITS + * FITNESS FOR ANY PARTICULAR PURPOSE. + * + * All of which is to say that you can do what you like with this + * source code provided you don't try to sell it as your own and you + * include an unaltered copy of this message (including the + * copyright). + * + * It is also implicitly understood that bug fixes and improvements + * should make their way back to the general Internet community so + * that everyone benefits. + */ + +``` + +# fiat +``` +The Apache License, Version 2.0 (Apache-2.0) + +Copyright 2015-2020 the fiat-crypto authors (see the AUTHORS file) + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. + +``` + +# g711 +``` +/* + * SpanDSP - a series of DSP components for telephony + * + * g711.h - In line A-law and u-law conversion routines + * + * Written by Steve Underwood <steveu@coppice.org> + * + * Copyright (C) 2001 Steve Underwood + * + * Despite my general liking of the GPL, I place this code in the + * public domain for the benefit of all mankind - even the slimy + * ones who might try to proprietize my work and use it to my + * detriment. + */ + +``` + +# g722 +``` +/* + * SpanDSP - a series of DSP components for telephony + * + * g722_decode.c - The ITU G.722 codec, decode part. + * + * Written by Steve Underwood <steveu@coppice.org> + * + * Copyright (C) 2005 Steve Underwood + * + * Despite my general liking of the GPL, I place my own contributions + * to this code in the public domain for the benefit of all mankind - + * even the slimy ones who might try to proprietize my work and use it + * to my detriment. + * + * Based in part on a single channel G.722 codec which is: + * + * Copyright (c) CMU 1993 + * Computer Science, Speech Group + * Chengxiang Lu and Alex Hauptmann + */ + +``` + +# libaom +``` +Copyright (c) 2016, Alliance for Open Media. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS +FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE +COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, +BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; +LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN +ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + +``` + +# libc++ +``` +============================================================================== +The LLVM Project is under the Apache License v2.0 with LLVM Exceptions: +============================================================================== + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +---- LLVM Exceptions to the Apache 2.0 License ---- + +As an exception, if, as a result of your compiling your source code, portions +of this Software are embedded into an Object form of such source code, you +may redistribute such embedded portions in such Object form without complying +with the conditions of Sections 4(a), 4(b) and 4(d) of the License. + +In addition, if you combine or link compiled forms of this Software with +software that is licensed under the GPLv2 ("Combined Software") and if a +court of competent jurisdiction determines that the patent provision (Section +3), the indemnity provision (Section 9) or other Section of the License +conflicts with the conditions of the GPLv2, you may retroactively and +prospectively choose to deem waived or otherwise exclude such Section(s) of +the License, but only in their entirety and only with respect to the Combined +Software. + +============================================================================== +Software from third parties included in the LLVM Project: +============================================================================== +The LLVM Project contains third party software which is under different license +terms. All such code will be identified clearly using at least one of two +mechanisms: +1) It will be in a separate directory tree with its own `LICENSE.txt` or + `LICENSE` file at the top containing the specific license and restrictions + which apply to that software, or +2) It will contain specific license and restriction terms at the top of every + file. + +============================================================================== +Legacy LLVM License (https://llvm.org/docs/DeveloperPolicy.html#legacy): +============================================================================== + +The libc++ library is dual licensed under both the University of Illinois +"BSD-Like" license and the MIT license. As a user of this code you may choose +to use it under either license. As a contributor, you agree to allow your code +to be used under both. + +Full text of the relevant licenses is included below. + +============================================================================== + +University of Illinois/NCSA +Open Source License + +Copyright (c) 2009-2019 by the contributors listed in CREDITS.TXT + +All rights reserved. + +Developed by: + + LLVM Team + + University of Illinois at Urbana-Champaign + + http://llvm.org + +Permission is hereby granted, free of charge, to any person obtaining a copy of +this software and associated documentation files (the "Software"), to deal with +the Software without restriction, including without limitation the rights to +use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +of the Software, and to permit persons to whom the Software is furnished to do +so, subject to the following conditions: + + * Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimers. + + * Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimers in the + documentation and/or other materials provided with the distribution. + + * Neither the names of the LLVM Team, University of Illinois at + Urbana-Champaign, nor the names of its contributors may be used to + endorse or promote products derived from this Software without specific + prior written permission. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS +FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +CONTRIBUTORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +============================================================================== + +Copyright (c) 2009-2014 by the contributors listed in CREDITS.TXT + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in +all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN +THE SOFTWARE. + +``` + +# libjpeg_turbo +``` +libjpeg-turbo Licenses +====================== + +libjpeg-turbo is covered by two compatible BSD-style open source licenses: + +- The IJG (Independent JPEG Group) License, which is listed in + [README.ijg](README.ijg) + + This license applies to the libjpeg API library and associated programs, + including any code inherited from libjpeg and any modifications to that + code. Note that the libjpeg-turbo SIMD source code bears the + [zlib License](https://opensource.org/licenses/Zlib), but in the context of + the overall libjpeg API library, the terms of the zlib License are subsumed + by the terms of the IJG License. + +- The Modified (3-clause) BSD License, which is listed below + + This license applies to the TurboJPEG API library and associated programs, as + well as the build system. Note that the TurboJPEG API library wraps the + libjpeg API library, so in the context of the overall TurboJPEG API library, + both the terms of the IJG License and the terms of the Modified (3-clause) + BSD License apply. + + +Complying with the libjpeg-turbo Licenses +========================================= + +This section provides a roll-up of the libjpeg-turbo licensing terms, to the +best of our understanding. This is not a license in and of itself. It is +intended solely for clarification. + +1. If you are distributing a modified version of the libjpeg-turbo source, + then: + + 1. You cannot alter or remove any existing copyright or license notices + from the source. + + **Origin** + - Clause 1 of the IJG License + - Clause 1 of the Modified BSD License + - Clauses 1 and 3 of the zlib License + + 2. You must add your own copyright notice to the header of each source + file you modified, so others can tell that you modified that file. (If + there is not an existing copyright header in that file, then you can + simply add a notice stating that you modified the file.) + + **Origin** + - Clause 1 of the IJG License + - Clause 2 of the zlib License + + 3. You must include the IJG README file, and you must not alter any of the + copyright or license text in that file. + + **Origin** + - Clause 1 of the IJG License + +2. If you are distributing only libjpeg-turbo binaries without the source, or + if you are distributing an application that statically links with + libjpeg-turbo, then: + + 1. Your product documentation must include a message stating: + + This software is based in part on the work of the Independent JPEG + Group. + + **Origin** + - Clause 2 of the IJG license + + 2. If your binary distribution includes or uses the TurboJPEG API, then + your product documentation must include the text of the Modified BSD + License (see below.) + + **Origin** + - Clause 2 of the Modified BSD License + +3. You cannot use the name of the IJG or The libjpeg-turbo Project or the + contributors thereof in advertising, publicity, etc. + + **Origin** + - IJG License + - Clause 3 of the Modified BSD License + +4. The IJG and The libjpeg-turbo Project do not warrant libjpeg-turbo to be + free of defects, nor do we accept any liability for undesirable + consequences resulting from your use of the software. + + **Origin** + - IJG License + - Modified BSD License + - zlib License + + +The Modified (3-clause) BSD License +=================================== + +Copyright (C)2009-2024 D. R. Commander. All Rights Reserved.<br> +Copyright (C)2015 Viktor Szathmáry. All Rights Reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + +- Redistributions of source code must retain the above copyright notice, + this list of conditions and the following disclaimer. +- Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. +- Neither the name of the libjpeg-turbo Project nor the names of its + contributors may be used to endorse or promote products derived from this + software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS", +AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDERS OR CONTRIBUTORS BE +LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR +CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF +SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS +INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN +CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) +ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE +POSSIBILITY OF SUCH DAMAGE. + + +Why Two Licenses? +================= + +The zlib License could have been used instead of the Modified (3-clause) BSD +License, and since the IJG License effectively subsumes the distribution +conditions of the zlib License, this would have effectively placed +libjpeg-turbo binary distributions under the IJG License. However, the IJG +License specifically refers to the Independent JPEG Group and does not extend +attribution and endorsement protections to other entities. Thus, it was +desirable to choose a license that granted us the same protections for new code +that were granted to the IJG for code derived from their software. + +``` + +# libsrtp +``` +/* + * + * Copyright (c) 2001-2017 Cisco Systems, Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following + * disclaimer in the documentation and/or other materials provided + * with the distribution. + * + * Neither the name of the Cisco Systems, Inc. nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS + * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE + * COPYRIGHT HOLDERS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, + * INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR + * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, + * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED + * OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +``` + +# libvpx +``` +Copyright (c) 2010, The WebM Project authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google, nor the WebM Project, nor the names + of its contributors may be used to endorse or promote products + derived from this software without specific prior written + permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +``` + +# libyuv +``` +Copyright 2011 The LibYuv Project Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + + * Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in + the documentation and/or other materials provided with the + distribution. + + * Neither the name of Google nor the names of its contributors may + be used to endorse or promote products derived from this software + without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` + +# nasm +``` +NASM is now licensed under the 2-clause BSD license, also known as the +simplified BSD license. + +SPDX-License-Identifier: BSD-2-Clause + + Copyright 1996-2025 the NASM Authors - All rights reserved. + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following + conditions are met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following + disclaimer in the documentation and/or other materials provided + with the distribution. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND + CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, + INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF + MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR + CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR + OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, + EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` + +# ooura +``` +/* + * http://www.kurims.kyoto-u.ac.jp/~ooura/fft.html + * Copyright Takuya OOURA, 1996-2001 + * + * You may use, copy, modify and distribute this code for any purpose (include + * commercial use) and without fee. Please refer to this package when you modify + * this code. + */ + +``` + +# opus +``` +Copyright 2001-2023 Xiph.Org, Skype Limited, Octasic, + Jean-Marc Valin, Timothy B. Terriberry, + CSIRO, Gregory Maxwell, Mark Borgerding, + Erik de Castro Lopo, Mozilla, Amazon + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +- Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +- Neither the name of Internet Society, IETF or IETF Trust, nor the +names of specific contributors, may be used to endorse or promote +products derived from this software without specific prior written +permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Opus is subject to the royalty-free patent licenses which are +specified at: + +Xiph.Org Foundation: +https://datatracker.ietf.org/ipr/1524/ + +Microsoft Corporation: +https://datatracker.ietf.org/ipr/1914/ + +Broadcom Corporation: +https://datatracker.ietf.org/ipr/1526/ + +``` + +# perfetto +``` + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + +------------------ + +Files: * except those files noted below + + Copyright (c) 2017, The Android Open Source Project + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +------------------ + +Files: src/trace_processor/perfetto_sql/stdlib/chromium/*, protos/third_party/chromium/*, test/trace_processor/diff_tests/stdlib/chrome/* + + Copyright 2015 The Chromium Authors + + Redistribution and use in source and binary forms, with or without + modification, are permitted provided that the following conditions are + met: + + * Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above + copyright notice, this list of conditions and the following disclaimer + in the documentation and/or other materials provided with the + distribution. + * Neither the name of Google LLC nor the names of its + contributors may be used to endorse or promote products derived from + this software without specific prior written permission. + + THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + + +------------------ + +Files: src/trace_processor/perfetto_sql/syntaqlite/syntaqlite_perfetto.{c, h} + + Copyright 2025 The syntaqlite Authors. All rights reserved. + + Machine-generated amalgamation of syntaqlite runtime + Perfetto dialect + sources (https://github.com/LalitMaganti/syntaqlite). Portions derive + from SQLite's public-domain `parse.y` grammar. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. + + +------------------ + +Files: src/trace_processor/perfetto_sql/preprocessor/preprocessor_grammar.{c, h} + +The author disclaims copyright to this source code. In place of a legal notice, here is a blessing: + +May you do good and not evil. +May you find forgiveness for yourself and forgive others. +May you share freely, never taking more than you give. + + +------------------ + +Files: src/base/intrusive_tree.cc + +$OpenBSD: tree.h,v 1.31 2023/03/08 04:43:09 guenther Exp $ + +Copyright 2002 Niels Provos <provos@citi.umich.edu> +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: +1. Redistributions of source code must retain the above copyright + notice, this list of conditions and the following disclaimer. +2. Redistributions in binary form must reproduce the above copyright + notice, this list of conditions and the following disclaimer in the + documentation and/or other materials provided with the distribution. + +THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR +IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES +OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. +IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, +INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT +NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF +THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` + +# pffft +``` +Copyright (c) 2013 Julien Pommier ( pommier@modartt.com ) + +Based on original fortran 77 code from FFTPACKv4 from NETLIB, +authored by Dr Paul Swarztrauber of NCAR, in 1985. + +As confirmed by the NCAR fftpack software curators, the following +FFTPACKv5 license applies to FFTPACKv4 sources. My changes are +released under the same terms. + +FFTPACK license: + +http://www.cisl.ucar.edu/css/software/fftpack5/ftpk.html + +Copyright (c) 2004 the University Corporation for Atmospheric +Research ("UCAR"). All rights reserved. Developed by NCAR's +Computational and Information Systems Laboratory, UCAR, +www.cisl.ucar.edu. + +Redistribution and use of the Software in source and binary forms, +with or without modification, is permitted provided that the +following conditions are met: + +- Neither the names of NCAR's Computational and Information Systems +Laboratory, the University Corporation for Atmospheric Research, +nor the names of its sponsors or contributors may be used to +endorse or promote products derived from this Software without +specific prior written permission. + +- Redistributions of source code must retain the above copyright +notices, this list of conditions, and the disclaimer below. + +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions, and the disclaimer below in the +documentation and/or other materials provided with the +distribution. + +THIS SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING, BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE CONTRIBUTORS OR COPYRIGHT +HOLDERS BE LIABLE FOR ANY CLAIM, INDIRECT, INCIDENTAL, SPECIAL, +EXEMPLARY, OR CONSEQUENTIAL DAMAGES OR OTHER LIABILITY, WHETHER IN AN +ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS WITH THE +SOFTWARE. + +``` + +# protobuf +``` +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. + +``` + +# rnnoise +``` +Copyright (c) 2017, Mozilla +Copyright (c) 2007-2017, Jean-Marc Valin +Copyright (c) 2005-2017, Xiph.Org Foundation +Copyright (c) 2003-2004, Mark Borgerding + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions +are met: + +- Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + +- Redistributions in binary form must reproduce the above copyright +notice, this list of conditions and the following disclaimer in the +documentation and/or other materials provided with the distribution. + +- Neither the name of the Xiph.Org Foundation nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION +OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +``` + +# spl_sqrt_floor +``` +/* + * Written by Wilco Dijkstra, 1996. The following email exchange establishes the + * license. + * + * From: Wilco Dijkstra <Wilco.Dijkstra@ntlworld.com> + * Date: Fri, Jun 24, 2011 at 3:20 AM + * Subject: Re: sqrt routine + * To: Kevin Ma <kma@google.com> + * Hi Kevin, + * Thanks for asking. Those routines are public domain (originally posted to + * comp.sys.arm a long time ago), so you can use them freely for any purpose. + * Cheers, + * Wilco + * + * ----- Original Message ----- + * From: "Kevin Ma" <kma@google.com> + * To: <Wilco.Dijkstra@ntlworld.com> + * Sent: Thursday, June 23, 2011 11:44 PM + * Subject: Fwd: sqrt routine + * Hi Wilco, + * I saw your sqrt routine from several web sites, including + * http://www.finesse.demon.co.uk/steven/sqrt.html. + * Just wonder if there's any copyright information with your Successive + * approximation routines, or if I can freely use it for any purpose. + * Thanks. + * Kevin + */ + +``` + +# zlib +``` +Copyright notice: + + (C) 1995-2026 Jean-loup Gailly and Mark Adler + + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. + + Jean-loup Gailly Mark Adler + jloup@gzip.org madler@alumni.caltech.edu + +``` diff --git a/native/capture/darwin/main.swift b/native/capture/darwin/main.swift index deee3de..bd2fa72 100644 --- a/native/capture/darwin/main.swift +++ b/native/capture/darwin/main.swift @@ -6,12 +6,13 @@ import Foundation import ScreenCaptureKit import VideoToolbox -private let captureProtocol = 4 +private let captureProtocol = 7 +private let maxOutputs = 6 private let width = 1280 private let height = 720 private let frameRate: Int32 = 30 private let bitrate = 3_000_000 -private let maxPayloadBytes = 1_048_576 +private let maxPayloadBytes = 4_194_304 private let supportedH264Levels: Set = [ 0x1e, 0x1f, 0x20, 0x28, 0x29, 0x2a, 0x32, 0x33, ] @@ -33,6 +34,21 @@ private struct VideoProfile { var timestampStep: CMTime { CMTime(value: 1, timescale: 90_000) } } +private struct OutputProfile: Codable { + let width: Int + let height: Int + let fps: Int32 + let bitrate: Int + + var frameInterval: CMTime { CMTime(value: 1, timescale: fps) } +} + +private enum CaptureControl { + case keyFrame(Int) + case active(Int, Bool) + case bitrate(Int, Int) +} + private let defaultVideoProfile = VideoProfile( width: width, height: height, @@ -100,6 +116,7 @@ private struct StartingStatus: Codable { let encoderIndex: UInt32 = 0 let encoderName = "VideoToolbox H.264" let encoderIdentity = "com.apple.videotoolbox.h264" + let outputs: [OutputProfile] } private struct ActiveStatus: Codable { @@ -110,6 +127,7 @@ private struct ActiveStatus: Codable { let width: Int let height: Int let fps: Int32 + let outputs: [OutputProfile] } private struct AudioStatus: Codable { @@ -300,18 +318,34 @@ private final class ProtocolWriter { func writeH264( _ payload: Data, keyFrame: Bool, + layer: UInt8, + profile: OutputProfile, timestamp: UInt64, duration: UInt64 ) throws { try write( kind: 2, flags: keyFrame ? 1 : 0, + layer: layer, + width: UInt16(profile.width), + height: UInt16(profile.height), timestamp: timestamp, duration: duration, payload: payload ) } + func beginFrame(timestamp: UInt64, duration: UInt64) throws { + try write(kind: 5, flags: 0, timestamp: timestamp, + duration: duration, payload: Data()) + } + + func unavailable(layer: UInt8, error: Error) throws { + let message = String(String(describing: error).prefix(256)) + try write(kind: 6, flags: 0, layer: layer, timestamp: 0, + duration: 0, payload: Data(message.utf8)) + } + func writePCM( _ payload: Data, timestamp: UInt64, @@ -329,16 +363,22 @@ private final class ProtocolWriter { private func write( kind: UInt8, flags: UInt8, + layer: UInt8 = 0, + width: UInt16 = 0, + height: UInt16 = 0, timestamp: UInt64, duration: UInt64, payload: Data ) throws { - guard !payload.isEmpty, payload.count <= maxPayloadBytes else { + let maximum = (kind == 3 || kind == 6) ? 1_048_576 : maxPayloadBytes + guard (kind == 5 || !payload.isEmpty), payload.count <= maximum else { throw CaptureFailure(description: "capture payload is outside its bound") } - var envelope = Data([0x53, 0x4d, 0x45, 0x44, 1, kind, flags, 0]) + var envelope = Data([0x53, 0x4d, 0x45, 0x44, 2, kind, flags, layer]) appendBigEndian(timestamp, to: &envelope) appendBigEndian(duration, to: &envelope) + appendBigEndian(width, to: &envelope) + appendBigEndian(height, to: &envelope) appendBigEndian(UInt32(payload.count), to: &envelope) envelope.append(payload) lock.lock() @@ -402,7 +442,7 @@ private func h264ParameterSets(_ format: CMFormatDescription) throws -> [Data] { return result } -private func annexB(_ sample: CMSampleBuffer, keyFrame: Bool) throws -> Data { +private func annexB(_ sample: CMSampleBuffer, keyFrame: Bool) throws -> (payload: Data, recovery: Bool) { guard let block = CMSampleBufferGetDataBuffer(sample), let format = CMSampleBufferGetFormatDescription(sample) else { throw CaptureFailure(description: "encoded H.264 sample is incomplete") @@ -431,6 +471,7 @@ private func annexB(_ sample: CMSampleBuffer, keyFrame: Bool) throws -> Data { } } var offset = 0 + var hasIDR = false while offset + 4 <= totalLength { let size = Int(bytes[offset]) << 24 | Int(bytes[offset + 1]) << 16 | @@ -440,6 +481,7 @@ private func annexB(_ sample: CMSampleBuffer, keyFrame: Bool) throws -> Data { guard size > 0, offset + size <= totalLength else { throw CaptureFailure(description: "invalid H.264 NAL length") } + hasIDR = hasIDR || bytes[offset] & 0x1f == 5 output.append(contentsOf: startCode) output.append(contentsOf: bytes[offset..<(offset + size)]) offset += size @@ -447,7 +489,7 @@ private func annexB(_ sample: CMSampleBuffer, keyFrame: Bool) throws -> Data { guard offset == totalLength else { throw CaptureFailure(description: "trailing H.264 sample bytes") } - return output + return (output, keyFrame && hasIDR) } private func h264ProfileLevelID(_ payload: Data) -> String? { @@ -468,26 +510,63 @@ private func h264ProfileLevelID(_ payload: Data) -> String? { } private final class HardwareEncoder { + private struct Input { + let image: CVImageBuffer + let timestamp: CMTime + } + private let writer: ProtocolWriter - private let done: StopSignal private let lock = NSLock() + private let queue: DispatchQueue private var session: VTCompressionSession? + private var transfer: VTPixelTransferSession? + private var pending: Input? + private var working = false + private var encodingTimestamp = CMTime.invalid + private var closed = false + private var failed = false private var forceKeyFrame = true - private var active = false - private var profileLevelId: String? - private let profile: VideoProfile + private var decodable = false + private var desiredBitrate: Int + private var currentBitrate: Int + private var lastFrameBucket: Int64? + private let sourceFrameRate: Int32 + private let profile: OutputProfile + private let layer: UInt8 + private let onActive: (String) throws -> Void + private let onFailure: (Error) -> Void init( writer: ProtocolWriter, done: StopSignal, - profile: VideoProfile = defaultVideoProfile + profile sourceProfile: VideoProfile = defaultVideoProfile, + output: OutputProfile? = nil, + layer: UInt8 = 0, + onActive: @escaping (String) throws -> Void = { _ in }, + onFailure: ((Error) -> Void)? = nil ) throws { self.writer = writer - self.done = done + let profile = output ?? OutputProfile( + width: sourceProfile.width, height: sourceProfile.height, + fps: sourceProfile.frameRate, bitrate: sourceProfile.bitrate + ) self.profile = profile + self.sourceFrameRate = sourceProfile.frameRate + self.layer = layer + self.onActive = onActive + self.onFailure = onFailure ?? { done.signal($0) } + self.queue = DispatchQueue(label: "screener.capture.encoder.\(layer)") + desiredBitrate = profile.bitrate + currentBitrate = profile.bitrate let specification = [ kVTVideoEncoderSpecification_RequireHardwareAcceleratedVideoEncoder as String: true, ] as CFDictionary + let attributes: [String: Any] = [ + kCVPixelBufferWidthKey as String: profile.width, + kCVPixelBufferHeightKey as String: profile.height, + kCVPixelBufferPixelFormatTypeKey as String: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, + kCVPixelBufferIOSurfacePropertiesKey as String: [:], + ] var created: VTCompressionSession? try require( VTCompressionSessionCreate( @@ -496,7 +575,7 @@ private final class HardwareEncoder { height: Int32(profile.height), codecType: kCMVideoCodecType_H264, encoderSpecification: specification, - imageBufferAttributes: nil, + imageBufferAttributes: attributes as CFDictionary, compressedDataAllocator: nil, outputCallback: nil, refcon: nil, @@ -508,21 +587,32 @@ private final class HardwareEncoder { throw CaptureFailure(description: "VideoToolbox returned no encoder") } session = created - try set(kVTCompressionPropertyKey_RealTime, kCFBooleanTrue) - try set(kVTCompressionPropertyKey_AllowFrameReordering, kCFBooleanFalse) - try set(kVTCompressionPropertyKey_ProfileLevel, - kVTProfileLevel_H264_ConstrainedBaseline_AutoLevel) - try set(kVTCompressionPropertyKey_AverageBitRate, NSNumber(value: profile.bitrate)) - try set(kVTCompressionPropertyKey_ExpectedFrameRate, NSNumber(value: profile.frameRate)) - try set(kVTCompressionPropertyKey_MaxKeyFrameInterval, NSNumber(value: profile.frameRate * 2)) - if profile.preference != .balanced { - try? set( - kVTCompressionPropertyKey_PrioritizeEncodingSpeedOverQuality, - profile.preference == .framerate ? kCFBooleanTrue : kCFBooleanFalse - ) + do { + try set(kVTCompressionPropertyKey_RealTime, kCFBooleanTrue) + try set(kVTCompressionPropertyKey_AllowFrameReordering, kCFBooleanFalse) + try set(kVTCompressionPropertyKey_ProfileLevel, + kVTProfileLevel_H264_ConstrainedBaseline_AutoLevel) + try set(kVTCompressionPropertyKey_AverageBitRate, NSNumber(value: profile.bitrate)) + try set(kVTCompressionPropertyKey_ExpectedFrameRate, NSNumber(value: profile.fps)) + try set(kVTCompressionPropertyKey_MaxKeyFrameInterval, NSNumber(value: profile.fps * 2)) + if sourceProfile.preference != .balanced { + try? set( + kVTCompressionPropertyKey_PrioritizeEncodingSpeedOverQuality, + sourceProfile.preference == .framerate ? kCFBooleanTrue : kCFBooleanFalse + ) + } + try require(VTPixelTransferSessionCreate( + allocator: nil, pixelTransferSessionOut: &transfer + ), "videotoolbox-scaler-create") + try require(VTCompressionSessionPrepareToEncodeFrames(created), + "videotoolbox-prepare") + } catch { + VTCompressionSessionInvalidate(created) + if let transfer { VTPixelTransferSessionInvalidate(transfer) } + session = nil + transfer = nil + throw error } - try require(VTCompressionSessionPrepareToEncodeFrames(created), - "videotoolbox-prepare") } private func set(_ key: CFString, _ value: CFTypeRef) throws { @@ -539,102 +629,324 @@ private final class HardwareEncoder { lock.unlock() } + func setBitrate(_ value: Int) { + lock.lock() + desiredBitrate = value + lock.unlock() + } + func encode(_ image: CVImageBuffer, timestamp: CMTime) { lock.lock() + guard !closed, !failed else { + lock.unlock() + return + } + if profile.fps < sourceFrameRate { + let bucket = CMTimeConvertScale( + timestamp, timescale: profile.fps, method: .roundTowardZero + ).value + if lastFrameBucket == bucket { + lock.unlock() + return + } + lastFrameBucket = bucket + } + // One retained input can wait behind one encode; newer raw frames replace it. + pending = Input(image: image, timestamp: timestamp) + let start = !working + working = true + lock.unlock() + if start { queue.async { self.encodePending() } } + } + + private func encodePending() { + lock.lock() + guard !closed, !failed, let input = pending else { + working = false + lock.unlock() + return + } + pending = nil let force = forceKeyFrame forceKeyFrame = false - let session = session + let bitrate = desiredBitrate + encodingTimestamp = input.timestamp lock.unlock() guard let session else { return } - let properties = force - ? [kVTEncodeFrameOptionKey_ForceKeyFrame as String: true] as CFDictionary - : nil - var synchronousFlags: VTEncodeInfoFlags = [] - let status = VTCompressionSessionEncodeFrame( - session, - imageBuffer: image, - presentationTimeStamp: timestamp, - duration: profile.frameInterval, - frameProperties: properties, - infoFlagsOut: &synchronousFlags - ) { [weak self] status, flags, sample in - guard let self else { return } - do { - try require(status, "videotoolbox-output") - if flags.contains(.frameDropped) { - if force { requestKeyFrame() } - return - } - guard let sample, CMSampleBufferDataIsReady(sample) else { - throw CaptureFailure(description: "VideoToolbox output is not ready") - } - let keyFrame = isKeyFrame(sample) - lock.lock() - let needsFirstKeyFrame = !active && !keyFrame - if (force && !keyFrame) || needsFirstKeyFrame { - forceKeyFrame = true - } - lock.unlock() - if needsFirstKeyFrame { return } - let payload = try annexB(sample, keyFrame: keyFrame) - let observedProfileLevelId = keyFrame - ? h264ProfileLevelID(payload) - : nil - let seconds = CMTimeGetSeconds(CMSampleBufferGetPresentationTimeStamp(sample)) - guard seconds.isFinite, seconds >= 0 else { - throw CaptureFailure(description: "invalid H.264 timestamp") - } - try writer.writeH264( - payload, - keyFrame: keyFrame, - timestamp: UInt64(seconds * 10_000_000), - duration: 10_000_000 / UInt64(profile.frameRate) - ) - lock.lock() - if let observedProfileLevelId { - profileLevelId = observedProfileLevelId - } - let publishActive = !active - let activeProfileLevelId = profileLevelId - if activeProfileLevelId != nil { - active = true - } - lock.unlock() - if publishActive { - guard let activeProfileLevelId else { - throw CaptureFailure(description: "H.264 key frame has no SPS profile") - } - try writer.writeStatus(ActiveStatus( - profileLevelId: activeProfileLevelId, - width: profile.width, - height: profile.height, - fps: profile.frameRate - )) - } - } catch { - done.signal(error) + do { + if currentBitrate != bitrate { + try set(kVTCompressionPropertyKey_AverageBitRate, NSNumber(value: bitrate)) + currentBitrate = bitrate } + let image = try scaled(input.image) + let properties = force + ? [kVTEncodeFrameOptionKey_ForceKeyFrame as String: true] as CFDictionary + : nil + var synchronousFlags: VTEncodeInfoFlags = [] + let status = VTCompressionSessionEncodeFrame( + session, + imageBuffer: image, + presentationTimeStamp: input.timestamp, + duration: profile.frameInterval, + frameProperties: properties, + infoFlagsOut: &synchronousFlags + ) { [self, image] status, flags, sample in + _ = image + defer { finish(input.timestamp) } + do { + try require(status, "videotoolbox-output") + if flags.contains(.frameDropped) { + if force { requestKeyFrame() } + return + } + guard let sample, CMSampleBufferDataIsReady(sample) else { + throw CaptureFailure(description: "VideoToolbox output is not ready") + } + let encoded = try annexB(sample, keyFrame: isKeyFrame(sample)) + let keyFrame = encoded.recovery + lock.lock() + let current = !closed && !failed && + CMTimeCompare(encodingTimestamp, input.timestamp) == 0 + let needsRecovery = !decodable && !keyFrame + if current && ((force && !keyFrame) || needsRecovery) { + forceKeyFrame = true + } + if current && keyFrame { decodable = true } + lock.unlock() + if !current || needsRecovery { return } + let payload = encoded.payload + let pts = CMSampleBufferGetPresentationTimeStamp(sample) + guard pts.isNumeric, CMTimeCompare(pts, .zero) >= 0 else { + throw CaptureFailure(description: "invalid H.264 timestamp") + } + try writer.writeH264( + payload, keyFrame: keyFrame, layer: layer, profile: profile, + timestamp: UInt64(CMTimeConvertScale( + pts, timescale: 10_000_000, method: .roundTowardZero + ).value), + duration: 10_000_000 / UInt64(profile.fps) + ) + if keyFrame { + guard let profileLevelId = h264ProfileLevelID(payload) else { + throw CaptureFailure(description: "H.264 key frame has no SPS profile") + } + try onActive(profileLevelId) + } + } catch { + if force { requestKeyFrame() } + fail(error) + } + } + try require(status, "videotoolbox-input") + if synchronousFlags.contains(.frameDropped) { + if force { requestKeyFrame() } + finish(input.timestamp) + } + } catch { + if force { requestKeyFrame() } + fail(error) + finish(input.timestamp) } - if status != noErr { - done.signal(CaptureFailure( - description: "videotoolbox-input failed (\(status))" - )) - } else if synchronousFlags.contains(.frameDropped), force { - requestKeyFrame() + } + + private func finish(_ timestamp: CMTime) { + lock.lock() + let current = encodingTimestamp.isValid && + CMTimeCompare(encodingTimestamp, timestamp) == 0 + if current { encodingTimestamp = .invalid } + lock.unlock() + if current { queue.async { self.encodePending() } } + } + + private func scaled(_ image: CVImageBuffer) throws -> CVPixelBuffer { + if CVPixelBufferGetWidth(image) == profile.width && + CVPixelBufferGetHeight(image) == profile.height { + return image } + guard let session, let transfer, + let pool = VTCompressionSessionGetPixelBufferPool(session) else { + throw CaptureFailure(description: "VideoToolbox scale buffers are unavailable") + } + var destination: CVPixelBuffer? + try require(CVPixelBufferPoolCreatePixelBuffer(nil, pool, &destination), + "videotoolbox-scale-buffer") + guard let destination else { + throw CaptureFailure(description: "VideoToolbox returned no scale buffer") + } + try require(VTPixelTransferSessionTransferImage( + transfer, from: image, to: destination + ), "videotoolbox-scale") + return destination + } + + private func fail(_ error: Error) { + lock.lock() + let first = !failed && !closed + failed = true + pending = nil + lock.unlock() + if first { onFailure(error) } } func close() { lock.lock() - let current = session - session = nil + let wasClosed = closed + closed = true + pending = nil lock.unlock() - guard let current else { return } - _ = VTCompressionSessionCompleteFrames( - current, - untilPresentationTimeStamp: .invalid + if wasClosed { return } + queue.sync { + if let session { + _ = VTCompressionSessionCompleteFrames( + session, untilPresentationTimeStamp: .invalid + ) + VTCompressionSessionInvalidate(session) + } + if let transfer { VTPixelTransferSessionInvalidate(transfer) } + session = nil + transfer = nil + } + } + + func flush() { + queue.sync { + if let session { + _ = VTCompressionSessionCompleteFrames( + session, untilPresentationTimeStamp: .invalid + ) + } + } + } +} + +private final class EncoderGroup { + private let writer: ProtocolWriter + private let done: StopSignal + private let profile: VideoProfile + private let outputs: [OutputProfile] + private let lock = NSLock() + // Encoder callbacks cannot wait behind synchronous encoder retirement. + private let lifecycle = NSLock() + private let originalOutput: Int + private var active = false + private var failedLayers = Set() + private var encoders: [HardwareEncoder?] = [] + private var bitrates: [Int] + private var closed = false + + init(writer: ProtocolWriter, done: StopSignal, profile: VideoProfile, + outputs: [OutputProfile], encoded: Bool = false) throws { + self.writer = writer + self.done = done + self.profile = profile + self.outputs = outputs + self.originalOutput = encoded ? 0 : min(1, outputs.count - 1) + self.bitrates = outputs.map(\.bitrate) + self.encoders = Array(repeating: nil, count: outputs.count) + try writer.writeStatus(StartingStatus(outputs: outputs)) + if !encoded { + for index in 0...originalOutput { setActive(index, true) } + if encoders[originalOutput] == nil { + close() + throw CaptureFailure(description: "original hardware video output is unavailable") + } + } + } + + private func reportActive(_ profileLevelId: String) throws { + lock.lock() + let publish = !active + active = true + lock.unlock() + if publish { + try writer.writeStatus(ActiveStatus( + profileLevelId: profileLevelId, width: profile.width, + height: profile.height, fps: profile.frameRate, outputs: outputs + )) + } + } + + private func unavailable(_ layer: UInt8, _ error: Error) { + lock.lock() + let first = failedLayers.insert(layer).inserted + let allFailed = failedLayers.count == outputs.count + let startupFailed = !active && Int(layer) == originalOutput + lock.unlock() + if first { + do { try writer.unavailable(layer: layer, error: error) } + catch { done.signal(error) } + } + if allFailed || startupFailed { done.signal(error) } + } + + func encode(_ image: CVImageBuffer, timestamp: CMTime) throws { + lifecycle.lock() + defer { lifecycle.unlock() } + if closed { return } + guard timestamp.isNumeric, CMTimeCompare(timestamp, .zero) >= 0 else { + throw CaptureFailure(description: "invalid capture timestamp") + } + try writer.beginFrame( + timestamp: UInt64(CMTimeConvertScale( + timestamp, timescale: 10_000_000, method: .roundTowardZero + ).value), + duration: 10_000_000 / UInt64(profile.frameRate) ) - VTCompressionSessionInvalidate(current) + for encoder in encoders { encoder?.encode(image, timestamp: timestamp) } + } + + func requestKeyFrame(_ layer: Int) { + lifecycle.lock() + defer { lifecycle.unlock() } + for (index, encoder) in encoders.enumerated() where layer == -1 || index == layer { + encoder?.requestKeyFrame() + } + } + + func setActive(_ index: Int, _ enabled: Bool) { + lifecycle.lock() + defer { lifecycle.unlock() } + if closed || enabled == (encoders[index] != nil) { return } + if !enabled { + let previous = encoders[index] + encoders[index] = nil + previous?.close() + return + } + let layer = UInt8(index) + lock.lock() + failedLayers.remove(layer) + lock.unlock() + do { + let encoder = try HardwareEncoder( + writer: writer, done: done, profile: profile, + output: outputs[index], layer: layer, + onActive: { [weak self] profileLevelId in + guard let self, index == self.originalOutput else { return } + try self.reportActive(profileLevelId) + }, + onFailure: { [weak self] in self?.unavailable(layer, $0) } + ) + encoder.setBitrate(bitrates[index]) + encoders[index] = encoder + } catch { + unavailable(layer, error) + } + } + + func setBitrate(_ layer: Int, _ bitrate: Int) { + lifecycle.lock() + defer { lifecycle.unlock() } + bitrates[layer] = bitrate + encoders[layer]?.setBitrate(bitrate) + } + + func close() { + lifecycle.lock() + defer { lifecycle.unlock() } + closed = true + for encoder in encoders { encoder?.close() } + encoders = Array(repeating: nil, count: outputs.count) } } @@ -694,11 +1006,11 @@ private func syntheticPixelBuffer() throws -> CVPixelBuffer { private func containsRequiredH264Units(_ envelope: Data) -> Bool { let bytes = [UInt8](envelope) - guard bytes.count > 32, bytes[5] == 2, bytes[6] & 1 == 1 else { + guard bytes.count >= 37, bytes[5] == 2, bytes[6] & 1 == 1 else { return false } var types = Set() - for index in 28..<(bytes.count - 4) where + for index in 32..<(bytes.count - 4) where bytes[index] == 0 && bytes[index + 1] == 0 && bytes[index + 2] == 0 && bytes[index + 3] == 1 { types.insert(bytes[index + 4] & 0x1f) @@ -714,6 +1026,7 @@ private func selfTest() throws { try syntheticPixelBuffer(), timestamp: CMTime(value: 1, timescale: frameRate) ) + encoder.flush() encoder.close() guard collector.snapshot().contains(where: containsRequiredH264Units) else { throw CaptureFailure(description: "hardware H.264 self-test produced no IDR") @@ -903,52 +1216,48 @@ private final class AudioCaptureOutput: NSObject, SCStreamOutput, SCStreamDelega } private final class CaptureOutput: NSObject, SCStreamOutput, SCStreamDelegate { - private let encoder: HardwareEncoder + private let encoders: EncoderGroup private let done: StopSignal private let queue: DispatchQueue private let profile: VideoProfile private var lastImage: CVImageBuffer? private var lastTimestamp = CMTime.invalid - private var lastRecoveryTime = CMTime.invalid init( - encoder: HardwareEncoder, + encoders: EncoderGroup, done: StopSignal, queue: DispatchQueue, profile: VideoProfile ) { - self.encoder = encoder + self.encoders = encoders self.done = done self.queue = queue self.profile = profile } - func requestKeyFrame() { - queue.async { [weak self] in - guard let self else { return } - encoder.requestKeyFrame() - guard let image = lastImage else { return } - let now = CMClockGetTime(CMClockGetHostTimeClock()) - guard now.isValid else { return } - if lastRecoveryTime.isValid, - CMTimeCompare( - now, - CMTimeAdd(lastRecoveryTime, profile.frameInterval) - ) < 0 { - return + func control(_ command: CaptureControl) { + queue.sync { + switch command { + case .keyFrame(let layer): encoders.requestKeyFrame(layer) + case .active(let slot, let enabled): encoders.setActive(slot, enabled) + case .bitrate(let layer, let bitrate): encoders.setBitrate(layer, bitrate) } - lastRecoveryTime = now - let next = lastTimestamp.isValid - ? later( - now, - CMTimeAdd(lastTimestamp, profile.timestampStep) - ) - : now - lastTimestamp = next - encoder.encode(image, timestamp: next) + do { try replayLastImage() } + catch { done.signal(error) } } } + private func replayLastImage() throws { + guard let image = lastImage else { return } + let now = CMClockGetTime(CMClockGetHostTimeClock()) + guard now.isNumeric else { return } + let timestamp = lastTimestamp.isValid + ? later(now, CMTimeAdd(lastTimestamp, profile.timestampStep)) + : now + lastTimestamp = timestamp + try encoders.encode(image, timestamp: timestamp) + } + func stream( _ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, @@ -976,7 +1285,8 @@ private final class CaptureOutput: NSObject, SCStreamOutput, SCStreamDelegate { : capturedTimestamp lastImage = image lastTimestamp = timestamp - encoder.encode(image, timestamp: timestamp) + do { try encoders.encode(image, timestamp: timestamp) } + catch { done.signal(error) } } func stream(_ stream: SCStream, didStopWithError error: Error) { @@ -1014,7 +1324,7 @@ private func captureFilter( } private func videoProfile(_ arguments: [String]) throws -> VideoProfile { - guard arguments.count == 23, + guard arguments.count >= 28, arguments[10] == "--width", let width = Int(arguments[11]), arguments[12] == "--height", let height = Int(arguments[13]), arguments[14] == "--fps", let frameRate = Int32(arguments[15]), @@ -1023,7 +1333,7 @@ private func videoProfile(_ arguments: [String]) throws -> VideoProfile { let preference = DegradationPreference(rawValue: arguments[19]), arguments[20] == "--codec", ["auto", "h264"].contains(arguments[21]), - arguments[22] == "--protocol-v4" else { + arguments[22] == "--protocol-v7" else { throw CaptureFailure(description: "invalid video profile arguments") } let validResolution = @@ -1044,8 +1354,291 @@ private func videoProfile(_ arguments: [String]) throws -> VideoProfile { ) } +private func outputProfiles(_ arguments: [String], start: Int = 23, + source: VideoProfile? = nil) throws -> [OutputProfile] { + let count = arguments.count - start + guard count > 0, count % 5 == 0, count / 5 <= maxOutputs else { + throw CaptureFailure(description: "invalid output profile count") + } + var result: [OutputProfile] = [] + for index in stride(from: start, to: arguments.count, by: 5) { + guard arguments[index] == "--output", + let width = Int(arguments[index + 1]), + let height = Int(arguments[index + 2]), + let fps = Int32(arguments[index + 3]), + let bitrate = Int(arguments[index + 4]), + width >= 2, width <= (source?.width ?? 2560), width % 2 == 0, + height >= 2, height <= (source?.height ?? 1440), height % 2 == 0, + fps >= 1, fps <= (source?.frameRate ?? 60), + bitrate >= 1000, bitrate <= (source?.bitrate ?? 12_000_000) else { + throw CaptureFailure(description: "output profile is outside source bounds") + } + result.append(OutputProfile(width: width, height: height, fps: fps, bitrate: bitrate)) + } + if let source { + let original = result[min(1, result.count - 1)] + guard original.width == source.width, + original.height == source.height, original.fps == source.frameRate, + original.bitrate == source.bitrate else { + throw CaptureFailure(description: "original output must match the source profile") + } + } + return result +} + +private struct EncodedInput { + let payload: Data + let timestamp: CMTime + let duration: CMTime +} + +private func readInput(done: StopSignal, outputs: [OutputProfile] = [], + handle: @escaping (CaptureControl) -> Void = { _ in }, + video: ((EncodedInput) throws -> Void)? = nil) { + DispatchQueue.global(qos: .userInitiated).async { + func readExactly(_ count: Int) throws -> Data? { + var result = Data() + while result.count < count { + let part = FileHandle.standardInput.readData(ofLength: count - result.count) + if part.isEmpty { + if result.isEmpty { return nil } + throw CaptureFailure(description: "truncated capture control") + } + result.append(part) + } + return result + } + do { + while let header = try readExactly(32) { + func number(_ range: Range) -> UInt64 { + header[range].reduce(0) { ($0 << 8) | UInt64($1) } + } + let size = Int(number(28..<32)) + guard Array(header.prefix(5)) == [0x53, 0x4d, 0x45, 0x44, 2], size > 0 else { + throw CaptureFailure(description: "invalid native input envelope") + } + if header[5] == 2, let video { + let timestamp = number(8..<16), duration = number(16..<24) + let width = number(24..<26), height = number(26..<28) + guard size <= maxPayloadBytes, header[6] <= 1, header[7] == 0, + width >= 2, width <= 2560, width % 2 == 0, + height >= 2, height <= 1440, height % 2 == 0, + timestamp <= UInt64(Int64.max) / 100, + duration > 0, duration <= UInt64(Int64.max) / 100, + let payload = try readExactly(size) else { + throw CaptureFailure(description: "invalid encoded input frame") + } + try video(EncodedInput(payload: payload, + timestamp: CMTime(value: Int64(timestamp), timescale: 10_000_000), + duration: CMTime(value: Int64(duration), timescale: 10_000_000))) + continue + } + guard header[5] == 7, header[6..<28].allSatisfy({ $0 == 0 }), + size <= 64, let payload = try readExactly(size), + payload.allSatisfy({ $0 >= 32 && $0 <= 126 }) else { + throw CaptureFailure(description: "invalid native control envelope") + } + let fields = String(decoding: payload, as: UTF8.self).split(whereSeparator: \.isWhitespace) + if fields == ["Q"] { + done.signal() + return + } + var command: CaptureControl? + if fields.count == 2, let value = Int(fields[1]) { + if fields[0] == "K", value >= -1, value < outputs.count { + command = .keyFrame(value) + } + } else if fields.count == 3, fields[0] == "A", + let slot = Int(fields[1]), let enabled = Int(fields[2]), + outputs.indices.contains(slot), (0...1).contains(enabled) { + command = .active(slot, enabled == 1) + } else if fields.count == 3, fields[0] == "B", + let layer = Int(fields[1]), let bitrate = Int(fields[2]), + outputs.indices.contains(layer), + bitrate >= 1000, bitrate <= outputs[layer].bitrate { + command = .bitrate(layer, bitrate) + } + guard let command, !outputs.isEmpty else { + done.signal(CaptureFailure(description: "invalid capture control")) + return + } + handle(command) + } + done.signal() + } catch { + done.signal(error) + } + } +} + +private final class H264InputDecoder { + private let encoders: EncoderGroup + private let lock = NSLock() + private var closed = false + private var session: VTDecompressionSession? + private var format: CMVideoFormatDescription? + private var sps = Data() + private var pps = Data() + private var lastTimestamp = CMTime.invalid + + init(encoders: EncoderGroup) { self.encoders = encoders } + + func close() { + lock.lock() + defer { lock.unlock() } + closed = true + if let session { VTDecompressionSessionInvalidate(session) } + session = nil + } + + func decode(_ input: EncodedInput) throws { + lock.lock() + defer { lock.unlock() } + guard !closed else { throw CaptureFailure(description: "encoded input is closed") } + guard !lastTimestamp.isValid || CMTimeCompare(input.timestamp, lastTimestamp) > 0 else { + throw CaptureFailure(description: "encoded input timestamp did not advance") + } + let bytes = [UInt8](input.payload) + func nextPrefix(_ start: Int) -> (offset: Int, size: Int)? { + var offset = start + while offset + 3 <= bytes.count { + if bytes[offset] == 0 && bytes[offset + 1] == 0 { + if bytes[offset + 2] == 1 { return (offset, 3) } + if offset + 4 <= bytes.count && bytes[offset + 2] == 0 && bytes[offset + 3] == 1 { + return (offset, 4) + } + } + offset += 1 + } + return nil + } + guard var prefix = nextPrefix(0), prefix.offset == 0 else { + throw CaptureFailure(description: "encoded H.264 input is not Annex-B") + } + var payload = Data(), nextSPS = sps, nextPPS = pps + var recovery = false + while true { + let start = prefix.offset + prefix.size + let next = nextPrefix(start) + let end = next?.offset ?? bytes.count + guard start < end else { throw CaptureFailure(description: "empty input NAL") } + let nal = Data(bytes[start..= 2, dimensions.width <= 2560, + dimensions.height >= 2, dimensions.height <= 1440 else { + throw CaptureFailure(description: "decoded input exceeds its dimension bound") + } + if let session, !VTDecompressionSessionCanAcceptFormatDescription(session, formatDescription: nextFormat) { + VTDecompressionSessionInvalidate(session) + self.session = nil + } + format = nextFormat + sps = nextSPS + pps = nextPPS + } + guard let format else { throw CaptureFailure(description: "input format is unavailable") } + if session == nil { + let attributes: [CFString: Any] = [ + kCVPixelBufferPixelFormatTypeKey: kCVPixelFormatType_420YpCbCr8BiPlanarVideoRange, + kCVPixelBufferIOSurfacePropertiesKey: [:], + ] + try require(VTDecompressionSessionCreate(allocator: kCFAllocatorDefault, + formatDescription: format, decoderSpecification: nil, + imageBufferAttributes: attributes as CFDictionary, outputCallback: nil, + decompressionSessionOut: &session), "decode-session") + } + var block: CMBlockBuffer? + try require(CMBlockBufferCreateWithMemoryBlock(allocator: kCFAllocatorDefault, + memoryBlock: nil, blockLength: payload.count, blockAllocator: kCFAllocatorDefault, + customBlockSource: nil, offsetToData: 0, dataLength: payload.count, + flags: 0, blockBufferOut: &block), "decode-block") + guard let block, let session else { throw CaptureFailure(description: "decoder is unavailable") } + try require(payload.withUnsafeBytes { bytes in + CMBlockBufferReplaceDataBytes(with: bytes.baseAddress!, blockBuffer: block, + offsetIntoDestination: 0, dataLength: payload.count) + }, "decode-payload") + var timing = CMSampleTimingInfo(duration: input.duration, + presentationTimeStamp: input.timestamp, decodeTimeStamp: .invalid) + var size = payload.count + var sample: CMSampleBuffer? + try require(CMSampleBufferCreateReady(allocator: kCFAllocatorDefault, dataBuffer: block, + formatDescription: format, sampleCount: 1, sampleTimingEntryCount: 1, + sampleTimingArray: &timing, sampleSizeEntryCount: 1, sampleSizeArray: &size, + sampleBufferOut: &sample), "decode-sample") + guard let sample else { throw CaptureFailure(description: "missing decoder sample") } + var image: CVImageBuffer? + var decodeStatus: OSStatus = noErr + // No async/reordering flags: VideoToolbox completes this callback before returning. + try require(VTDecompressionSessionDecodeFrame(session, sampleBuffer: sample, + flags: [], infoFlagsOut: nil, outputHandler: { status, _, decoded, _, _ in + decodeStatus = status + image = decoded + }), "decode-frame") + try require(decodeStatus, "decode-output") + guard let image else { throw CaptureFailure(description: "decoder did not produce an image") } + lastTimestamp = input.timestamp + try encoders.encode(image, timestamp: input.timestamp) + } +} + +private func encodedVideo(_ arguments: [String]) throws { + guard arguments.count >= 16, arguments[2] == "--codec", arguments[3] == "h264", + arguments[4] == "--adapter-index", arguments[5] == "0", + arguments[6] == "--mft-index", arguments[7] == "0", + arguments[8] == "--preference", + let preference = DegradationPreference(rawValue: arguments[9]), + arguments[10] == "--protocol-v7" else { + throw CaptureFailure(description: "invalid encoded video arguments") + } + let outputs = try outputProfiles(arguments, start: 11) + let profile = VideoProfile(width: outputs.map(\.width).max()!, height: outputs.map(\.height).max()!, + frameRate: outputs.map(\.fps).max()!, bitrate: outputs.map(\.bitrate).max()!, preference: preference) + let done = StopSignal() + let encoders = try EncoderGroup(writer: ProtocolWriter(), done: done, profile: profile, + outputs: outputs, encoded: true) + defer { encoders.close() } + let decoder = H264InputDecoder(encoders: encoders) + defer { decoder.close() } + readInput(done: done, outputs: outputs, handle: { command in + switch command { + case .keyFrame(let layer): encoders.requestKeyFrame(layer) + case .active(let slot, let enabled): encoders.setActive(slot, enabled) + case .bitrate(let layer, let bitrate): encoders.setBitrate(layer, bitrate) + } + }, video: decoder.decode) + try done.wait() +} + private func capture(_ arguments: [String]) async throws { - guard arguments.count == 23, + guard arguments.count >= 28, arguments[1] == "--capture-video", let sourceID = UInt32(arguments[3]), sourceID > 0, let pid = UInt32(arguments[4]), @@ -1054,6 +1647,7 @@ private func capture(_ arguments: [String]) async throws { throw CaptureFailure(description: "invalid capture arguments") } let profile = try videoProfile(arguments) + let profiles = try outputProfiles(arguments, source: profile) let kind = arguments[2] let creationTime = arguments[5] let filter = try await captureFilter( @@ -1065,11 +1659,11 @@ private func capture(_ arguments: [String]) async throws { let writer = ProtocolWriter() let done = StopSignal() - let encoder = try HardwareEncoder(writer: writer, done: done, profile: profile) - defer { encoder.close() } + let encoders = try EncoderGroup(writer: writer, done: done, profile: profile, outputs: profiles) + defer { encoders.close() } let captureQueue = DispatchQueue(label: "screener.capture.video") let output = CaptureOutput( - encoder: encoder, + encoders: encoders, done: done, queue: captureQueue, profile: profile @@ -1091,36 +1685,16 @@ private func capture(_ arguments: [String]) async throws { type: .screen, sampleHandlerQueue: captureQueue ) - try writer.writeStatus(StartingStatus()) - - DispatchQueue.global(qos: .userInitiated).async { - while true { - let data = FileHandle.standardInput.readData(ofLength: 64) - if data.isEmpty { - done.signal() - return - } - for byte in data { - if byte == 75 { - output.requestKeyFrame() - } else if byte == 81 || byte == 10 || byte == 13 { - done.signal() - return - } - } - } - } + readInput(done: done, outputs: profiles, handle: output.control) try await stream.startCapture() do { try done.wait() } catch { try? await stream.stopCapture() - encoder.close() throw error } try await stream.stopCapture() - encoder.close() } private func captureAudio(_ arguments: [String]) async throws { @@ -1166,16 +1740,7 @@ private func captureAudio(_ arguments: [String]) async throws { ) try stream.addStreamOutput(output, type: .audio, sampleHandlerQueue: queue) - DispatchQueue.global(qos: .userInitiated).async { - while true { - let data = FileHandle.standardInput.readData(ofLength: 64) - if data.isEmpty || data.contains(81) || data.contains(10) || - data.contains(13) { - done.signal() - return - } - } - } + readInput(done: done) try await stream.startCapture() do { @@ -1200,6 +1765,8 @@ private struct ScreenerCapture { try await listSources() } else if arguments.count > 1, arguments[1] == "--capture-audio" { try await captureAudio(arguments) + } else if arguments.count > 1, arguments[1] == "--encoded-video" { + try encodedVideo(arguments) } else { try await capture(arguments) } diff --git a/native/capture/linux/main.c b/native/capture/linux/main.c index 93157ca..102a7aa 100644 --- a/native/capture/linux/main.c +++ b/native/capture/linux/main.c @@ -1,6 +1,8 @@ #include +#include #include #include +#include #include #include @@ -13,7 +15,8 @@ #include "portal.h" enum { - kCaptureProtocol = 4, + kCaptureProtocol = 7, + kMaxOutputs = 6, kMaxPayloadBytes = 4 * 1024 * 1024, kAudioFrameBytes = 960 * 2 * 2, }; @@ -38,21 +41,50 @@ typedef struct { guint bit_rate_divisor; } EncoderInfo; +typedef struct CaptureRun CaptureRun; + typedef struct { + CaptureRun *run; + GstElement *encoder; + GstElement *branch; + GstElement *gate; + VideoProfile profile; + guint layer; + gboolean enabled; + gboolean decodable; + guint64 key_requested; + guint64 key_sent; + guint64 key_acknowledged; + GstClockTime key_timestamp; + GstClockTime activation_timestamp; +} VideoOutput; + +struct CaptureRun { GMainLoop *loop; GstElement *pipeline; - GstElement *encoder; + GstElement *encoded_source; VideoProfile profile; + VideoOutput outputs[kMaxOutputs]; + guint output_count; + guint original_output; const EncoderInfo *encoder_info; const char *restore_token; GMutex lock; char *failure; gint stopping; gboolean active; - guint64 last_timestamp; + GstClockTime last_input_timestamp; + guint8 input_header[32]; + guint input_header_length; + GByteArray *input_payload; + guint input_size; + guint64 encoded_timestamp; + gboolean encoded_started; GByteArray *audio; guint64 audio_timestamp; -} CaptureRun; +}; + +static GMutex output_lock; static gboolean write_all(const void *data, size_t size) { const uint8_t *next = data; @@ -79,15 +111,29 @@ static void put_u64_be(uint8_t *output, guint64 value) { } } -static gboolean write_frame(guint8 kind, guint8 flags, guint64 timestamp, - guint64 duration, const guint8 *payload, - gsize size) { - if (payload == NULL || size == 0 || size > kMaxPayloadBytes) return FALSE; - uint8_t header[28] = {'S', 'M', 'E', 'D', 1, kind, flags, 0}; +static guint64 read_be(const guint8 *input, guint count) { + guint64 value = 0; + for (guint index = 0; index < count; ++index) value = (value << 8) | input[index]; + return value; +} + +static gboolean write_frame(guint8 kind, guint8 flags, guint8 layer, + guint width, guint height, guint64 timestamp, + guint64 duration, const guint8 *payload, gsize size) { + if ((kind != 5 && (payload == NULL || size == 0)) || + size > ((kind == 3 || kind == 6) ? 1024 * 1024 : kMaxPayloadBytes)) return FALSE; + uint8_t header[32] = {'S', 'M', 'E', 'D', 2, kind, flags, layer}; put_u64_be(header + 8, timestamp); put_u64_be(header + 16, duration); - put_u32_be(header + 24, (guint32)size); - return write_all(header, sizeof(header)) && write_all(payload, size); + header[24] = (guint8)(width >> 8); + header[25] = (guint8)width; + header[26] = (guint8)(height >> 8); + header[27] = (guint8)height; + put_u32_be(header + 28, (guint32)size); + g_mutex_lock(&output_lock); + gboolean written = write_all(header, sizeof(header)) && write_all(payload, size); + g_mutex_unlock(&output_lock); + return written; } static char *json_string(const char *value) { @@ -129,7 +175,20 @@ static char *json_string(const char *value) { } static gboolean write_status(const char *json) { - return write_frame(3, 0, 0, 0, (const guint8 *)json, strlen(json)); + return write_frame(3, 0, 0, 0, 0, 0, 0, (const guint8 *)json, strlen(json)); +} + +static char *output_profiles_json(const CaptureRun *run) { + GString *json = g_string_new("["); + for (guint index = 0; index < run->output_count; ++index) { + const VideoProfile *profile = &run->outputs[index].profile; + g_string_append_printf(json, + "%s{\"width\":%u,\"height\":%u,\"fps\":%u,\"bitrate\":%u}", + index == 0 ? "" : ",", profile->width, profile->height, + profile->frame_rate, profile->bit_rate); + } + g_string_append_c(json, ']'); + return g_string_free(json, FALSE); } static void encoder_info_free(gpointer value) { @@ -226,7 +285,7 @@ static gboolean has_factory(const char *name) { } static gboolean video_stack_available(void) { - const char *required[] = {"pipewiresrc", "queue", "videoconvert", + const char *required[] = {"pipewiresrc", "queue", "tee", "videoconvert", "videoscale", "videorate", "h264parse", "appsink"}; for (guint index = 0; index < G_N_ELEMENTS(required); ++index) { @@ -309,9 +368,60 @@ static gboolean parse_uint(const char *value, guint minimum, guint maximum, return TRUE; } -static gboolean parse_profile(int count, char **values, VideoProfile *profile, +static gboolean parse_outputs(int count, char **values, guint start, CaptureRun *run) { + if (count <= (int)start || (count - (int)start) % 5 != 0 || + (count - (int)start) / 5 > kMaxOutputs) return FALSE; + run->output_count = (guint)(count - (int)start) / 5; + for (guint index = 0; index < run->output_count; ++index) { + guint offset = start + index * 5; + VideoOutput *output = &run->outputs[index]; + output->run = run; + output->layer = index; + output->enabled = strcmp(values[1], "--capture-video") == 0 && index < 2; + output->key_requested = 1; + output->key_timestamp = GST_CLOCK_TIME_NONE; + output->activation_timestamp = GST_CLOCK_TIME_NONE; + output->profile.preference = run->profile.preference; + if (strcmp(values[offset], "--output") != 0 || + !parse_uint(values[offset + 1], 2, 2560, &output->profile.width) || + !parse_uint(values[offset + 2], 2, 1440, &output->profile.height) || + !parse_uint(values[offset + 3], 1, 60, &output->profile.frame_rate) || + !parse_uint(values[offset + 4], 1000, 12000000, &output->profile.bit_rate) || + output->profile.width % 2 != 0 || output->profile.height % 2 != 0) return FALSE; + } + return TRUE; +} + +static gboolean parse_preference(const char *value, VideoProfile *profile) { + if (strcmp(value, "maintain-resolution") == 0) profile->preference = kPreferResolution; + else if (strcmp(value, "balanced") == 0) profile->preference = kPreferBalanced; + else if (strcmp(value, "maintain-framerate") == 0) profile->preference = kPreferFramerate; + else return FALSE; + return TRUE; +} + +static gboolean parse_profile(int count, char **values, CaptureRun *run, guint *encoder_index) { - if (count != 23 || strcmp(values[1], "--capture-video") != 0 || + VideoProfile *profile = &run->profile; + if (count >= 16 && strcmp(values[1], "--encoded-video") == 0) { + if (strcmp(values[2], "--codec") != 0 || strcmp(values[3], "h264") != 0 || + strcmp(values[4], "--adapter-index") != 0 || strcmp(values[5], "0") != 0 || + strcmp(values[6], "--mft-index") != 0 || !parse_uint(values[7], 0, 63, encoder_index) || + strcmp(values[8], "--preference") != 0 || !parse_preference(values[9], profile) || + strcmp(values[10], "--protocol-v7") != 0 || !parse_outputs(count, values, 11, run)) return FALSE; + for (guint index = 0; index < run->output_count; ++index) { + const VideoProfile *output = &run->outputs[index].profile; + profile->width = MAX(profile->width, output->width); + profile->height = MAX(profile->height, output->height); + profile->frame_rate = MAX(profile->frame_rate, output->frame_rate); + profile->bit_rate = MAX(profile->bit_rate, output->bit_rate); + } + run->original_output = 0; + return TRUE; + } + if (count < 28 || (count - 23) % 5 != 0 || + (count - 23) / 5 > kMaxOutputs || + strcmp(values[1], "--capture-video") != 0 || strcmp(values[2], "picker") != 0 || strcmp(values[3], "1") != 0 || strcmp(values[4], "0") != 0 || strcmp(values[5], "0") != 0 || strcmp(values[6], "--adapter-index") != 0 || @@ -323,7 +433,7 @@ static gboolean parse_profile(int count, char **values, VideoProfile *profile, strcmp(values[18], "--preference") != 0 || strcmp(values[20], "--codec") != 0 || (strcmp(values[21], "auto") != 0 && strcmp(values[21], "h264") != 0) || - strcmp(values[22], "--protocol-v4") != 0 || + strcmp(values[22], "--protocol-v7") != 0 || !parse_uint(values[9], 0, 63, encoder_index) || !parse_uint(values[11], 1, 16384, &profile->width) || !parse_uint(values[13], 1, 16384, &profile->height) || @@ -337,14 +447,16 @@ static gboolean parse_profile(int count, char **values, VideoProfile *profile, (profile->width == 1920 && profile->height == 1080) || (profile->width == 2560 && profile->height == 1440); if (!resolution) return FALSE; - if (strcmp(values[19], "maintain-resolution") == 0) { - profile->preference = kPreferResolution; - } else if (strcmp(values[19], "balanced") == 0) { - profile->preference = kPreferBalanced; - } else if (strcmp(values[19], "maintain-framerate") == 0) { - profile->preference = kPreferFramerate; - } else { - return FALSE; + if (!parse_preference(values[19], profile) || !parse_outputs(count, values, 23, run)) return FALSE; + run->original_output = MIN(1, run->output_count - 1); + const VideoProfile *original = &run->outputs[run->original_output].profile; + if (original->width != profile->width || original->height != profile->height || + original->frame_rate != profile->frame_rate || + original->bit_rate != profile->bit_rate) return FALSE; + for (guint index = 0; index < run->output_count; ++index) { + const VideoProfile *output = &run->outputs[index].profile; + if (output->width > profile->width || output->height > profile->height || + output->frame_rate > profile->frame_rate || output->bit_rate > profile->bit_rate) return FALSE; } return TRUE; } @@ -355,17 +467,26 @@ static gboolean set_numeric_property(GObject *object, const char *name, g_object_class_find_property(G_OBJECT_GET_CLASS(object), name); if (property == NULL || !(property->flags & G_PARAM_WRITABLE)) return FALSE; GType type = G_PARAM_SPEC_VALUE_TYPE(property); + GValue setting = G_VALUE_INIT; + g_value_init(&setting, type); if (type == G_TYPE_UINT) { - g_object_set(object, name, (guint)value, NULL); + g_value_set_uint(&setting, (guint)value); } else if (type == G_TYPE_INT) { - g_object_set(object, name, (gint)value, NULL); + g_value_set_int(&setting, (gint)value); } else if (type == G_TYPE_UINT64) { - g_object_set(object, name, value, NULL); + g_value_set_uint64(&setting, value); } else if (type == G_TYPE_INT64) { - g_object_set(object, name, (gint64)value, NULL); + g_value_set_int64(&setting, (gint64)value); } else { + g_value_unset(&setting); return FALSE; } + if (g_param_value_validate(property, &setting)) { + g_value_unset(&setting); + return FALSE; + } + g_object_set_property(object, name, &setting); + g_value_unset(&setting); return TRUE; } @@ -393,24 +514,25 @@ static void set_optional_enum(GObject *object, const char *name, g_type_class_unref(values); } -static gboolean configure_encoder(CaptureRun *run) { +static gboolean configure_encoder(VideoOutput *output) { + CaptureRun *run = output->run; guint64 bitrate = - run->profile.bit_rate / run->encoder_info->bit_rate_divisor; - if (!set_numeric_property(G_OBJECT(run->encoder), "bitrate", bitrate)) { + output->profile.bit_rate / run->encoder_info->bit_rate_divisor; + if (!set_numeric_property(G_OBJECT(output->encoder), "bitrate", bitrate)) { return FALSE; } - set_numeric_property(G_OBJECT(run->encoder), "key-int-max", - run->profile.frame_rate * 2); - set_numeric_property(G_OBJECT(run->encoder), "b-frames", 0); - set_numeric_property(G_OBJECT(run->encoder), "ref-frames", 1); - set_optional_boolean(G_OBJECT(run->encoder), "cabac", FALSE); - set_optional_boolean(G_OBJECT(run->encoder), "aud", TRUE); - set_optional_boolean(G_OBJECT(run->encoder), "zerolatency", TRUE); - set_optional_enum(G_OBJECT(run->encoder), "rate-control", "cbr"); - guint usage = run->profile.preference == kPreferResolution + set_numeric_property(G_OBJECT(output->encoder), "key-int-max", + output->profile.frame_rate * 2); + set_numeric_property(G_OBJECT(output->encoder), "b-frames", 0); + set_numeric_property(G_OBJECT(output->encoder), "ref-frames", 1); + set_optional_boolean(G_OBJECT(output->encoder), "cabac", FALSE); + set_optional_boolean(G_OBJECT(output->encoder), "aud", TRUE); + set_optional_boolean(G_OBJECT(output->encoder), "zerolatency", TRUE); + set_optional_enum(G_OBJECT(output->encoder), "rate-control", "cbr"); + guint usage = output->profile.preference == kPreferResolution ? 1 : run->profile.preference == kPreferFramerate ? 7 : 4; - set_numeric_property(G_OBJECT(run->encoder), "target-usage", usage); + set_numeric_property(G_OBJECT(output->encoder), "target-usage", usage); return TRUE; } @@ -480,40 +602,232 @@ static gboolean quit_loop(gpointer data) { static void fail_run(CaptureRun *run, const char *message) { g_mutex_lock(&run->lock); - if (run->failure == NULL) run->failure = g_strdup(message); + gboolean first = run->failure == NULL; + if (first) run->failure = g_strdup(message); g_mutex_unlock(&run->lock); - g_main_context_invoke(NULL, quit_loop, run); + if (first) g_main_context_invoke(NULL, quit_loop, run); } -static gboolean force_key_frame(gpointer data) { - CaptureRun *run = data; - if (run->encoder == NULL) return G_SOURCE_REMOVE; - GstPad *source = gst_element_get_static_pad(run->encoder, "src"); - if (source != NULL) { - gst_pad_send_event( - source, - gst_video_event_new_upstream_force_key_unit(GST_CLOCK_TIME_NONE, TRUE, - 0)); - gst_object_unref(source); +static void fail_output(VideoOutput *output, const char *message) { + write_frame(6, 0, (guint8)output->layer, 0, 0, 0, 0, + (const guint8 *)message, MIN(strlen(message), 512)); + fail_run(output->run, message); +} + +static gboolean set_output_active(VideoOutput *output, gboolean enabled) { + CaptureRun *run = output->run; + g_mutex_lock(&run->lock); + if (output->enabled == enabled) { + g_mutex_unlock(&run->lock); + return TRUE; } - return G_SOURCE_REMOVE; + output->enabled = FALSE; + output->decodable = FALSE; + output->activation_timestamp = GST_CLOCK_TIME_NONE; + output->key_timestamp = GST_CLOCK_TIME_NONE; + ++output->key_requested; + g_mutex_unlock(&run->lock); + // A closed valve isolates the retired branch from the other tee subscribers. + g_object_set(output->gate, "drop", TRUE, NULL); + gst_element_set_locked_state(output->branch, !enabled); + gboolean applied = enabled + ? gst_element_sync_state_with_parent(output->branch) + : gst_element_set_state(output->branch, GST_STATE_NULL) != GST_STATE_CHANGE_FAILURE; + if (!applied) return FALSE; + g_mutex_lock(&run->lock); + output->enabled = enabled; + g_mutex_unlock(&run->lock); + if (enabled) g_object_set(output->gate, "drop", FALSE, NULL); + return TRUE; } -static gpointer control_input(gpointer data) { +static gboolean apply_control(CaptureRun *run, char *line) { + char *values[4] = {0}; + guint count = 0; + g_auto(GStrv) tokens = g_strsplit_set(line, " \t\r", -1); + for (guint index = 0; tokens[index] != NULL; ++index) { + char *part = tokens[index]; + if (part[0] == '\0') continue; + if (count == G_N_ELEMENTS(values)) return FALSE; + values[count++] = part; + } + if (count == 1 && strcmp(values[0], "Q") == 0) { + g_atomic_int_set(&run->stopping, TRUE); + g_main_loop_quit(run->loop); + return TRUE; + } + if (run->output_count == 0) return FALSE; + guint layer = 0; + if (count == 2 && strcmp(values[0], "K") == 0) { + gboolean all = strcmp(values[1], "-1") == 0; + if (!all && !parse_uint(values[1], 0, run->output_count - 1, &layer)) return FALSE; + g_mutex_lock(&run->lock); + for (guint index = 0; index < run->output_count; ++index) { + if (all || index == layer) ++run->outputs[index].key_requested; + } + g_mutex_unlock(&run->lock); + return TRUE; + } + if (count == 3 && strcmp(values[0], "A") == 0) { + guint active = 0; + if (!parse_uint(values[1], 0, run->output_count - 1, &layer) || + !parse_uint(values[2], 0, 1, &active)) return FALSE; + return set_output_active(&run->outputs[layer], active != 0); + } + if (count == 3 && strcmp(values[0], "B") == 0 && + parse_uint(values[1], 0, run->output_count - 1, &layer)) { + VideoOutput *output = &run->outputs[layer]; + guint bitrate = 0; + if (!parse_uint(values[2], 1000, output->profile.bit_rate, &bitrate)) return FALSE; + GParamSpec *property = g_object_class_find_property( + G_OBJECT_GET_CLASS(output->encoder), "bitrate"); + if (property == NULL || !(property->flags & GST_PARAM_MUTABLE_PLAYING) || + !set_numeric_property(G_OBJECT(output->encoder), "bitrate", + MAX(1, bitrate / run->encoder_info->bit_rate_divisor))) { + fail_output(output, "hardware encoder cannot apply the live bitrate"); + } + return TRUE; + } + return FALSE; +} + +static gboolean valid_input_header(CaptureRun *run) { + const guint8 *header = run->input_header; + if (memcmp(header, "SMED", 4) != 0 || header[4] != 2) return FALSE; + run->input_size = (guint)read_be(header + 28, 4); + if (header[5] == 7) { + for (guint index = 6; index < 28; ++index) if (header[index] != 0) return FALSE; + return run->input_size > 0 && run->input_size <= 64; + } + if (header[5] != 2 || run->encoded_source == NULL || header[6] > 1 || header[7] != 0 || + run->input_size == 0 || run->input_size > kMaxPayloadBytes) return FALSE; + guint width = (guint)read_be(header + 24, 2), height = (guint)read_be(header + 26, 2); + guint64 timestamp = read_be(header + 8, 8), duration = read_be(header + 16, 8); + return width >= 2 && width <= 2560 && height >= 2 && height <= 1440 && + width % 2 == 0 && height % 2 == 0 && duration > 0 && + timestamp <= INT64_MAX / 100 && duration <= INT64_MAX / 100; +} + +static gboolean apply_input(CaptureRun *run) { + const guint8 *payload = run->input_payload->data; + if (run->input_header[5] == 7) { + for (guint index = 0; index < run->input_size; ++index) if (payload[index] < 32 || payload[index] > 126) return FALSE; + char command[65]; + memcpy(command, payload, run->input_size); + command[run->input_size] = '\0'; + return apply_control(run, command); + } + NalSummary nal = inspect_h264(payload, run->input_size); + guint64 timestamp = read_be(run->input_header + 8, 8); + if ((!run->encoded_started && !(nal.sps && nal.pps && nal.idr)) || + (run->encoded_started && timestamp <= run->encoded_timestamp)) return FALSE; + GstBuffer *buffer = gst_buffer_new_allocate(NULL, run->input_size, NULL); + if (buffer == NULL) return FALSE; + gst_buffer_fill(buffer, 0, payload, run->input_size); + GST_BUFFER_PTS(buffer) = timestamp * 100; + GST_BUFFER_DTS(buffer) = GST_CLOCK_TIME_NONE; + GST_BUFFER_DURATION(buffer) = read_be(run->input_header + 16, 8) * 100; + if (!nal.idr) GST_BUFFER_FLAG_SET(buffer, GST_BUFFER_FLAG_DELTA_UNIT); + run->encoded_started = TRUE; + run->encoded_timestamp = timestamp; + // appsrc blocks at one queued AU; compressed reference frames are never dropped. + return gst_app_src_push_buffer(GST_APP_SRC(run->encoded_source), buffer) == GST_FLOW_OK; +} + +static gboolean control_input(gint fd, GIOCondition condition, gpointer data) { CaptureRun *run = data; - guint8 value = 0; - while (read(STDIN_FILENO, &value, 1) == 1) { - if (value == 'K') { - g_main_context_invoke(NULL, force_key_frame, run); - } else if (value == 'Q' || value == '\n' || value == '\r') { - g_atomic_int_set(&run->stopping, TRUE); - g_main_context_invoke(NULL, quit_loop, run); - return NULL; + guint8 bytes[32 * 1024]; + ssize_t size = read(fd, bytes, sizeof(bytes)); + if (size < 0 && errno == EINTR) return G_SOURCE_CONTINUE; + if (size <= 0) { + g_atomic_int_set(&run->stopping, TRUE); + g_main_loop_quit(run->loop); + return G_SOURCE_CONTINUE; + } + (void)condition; + gsize offset = 0; + while (offset < (gsize)size && !g_atomic_int_get(&run->stopping)) { + if (run->input_header_length < 32) { + gsize take = MIN(32 - run->input_header_length, (gsize)size - offset); + memcpy(run->input_header + run->input_header_length, bytes + offset, take); + offset += take; + run->input_header_length += (guint)take; + if (run->input_header_length < 32) continue; + if (!valid_input_header(run)) { + fail_run(run, "invalid native input envelope"); + return G_SOURCE_CONTINUE; + } + } + gsize take = MIN(run->input_size - run->input_payload->len, (gsize)size - offset); + g_byte_array_append(run->input_payload, bytes + offset, (guint)take); + offset += take; + if (run->input_payload->len < run->input_size) continue; + if (!apply_input(run)) { + fail_run(run, "invalid or failed native input"); + return G_SOURCE_CONTINUE; + } + run->input_header_length = 0; + run->input_size = 0; + g_byte_array_set_size(run->input_payload, 0); + } + return G_SOURCE_CONTINUE; +} + +static GstPadProbeReturn begin_input_frame(GstPad *pad, GstPadProbeInfo *info, + gpointer data) { + (void)pad; + CaptureRun *run = data; + GstBuffer *buffer = GST_PAD_PROBE_INFO_BUFFER(info); + if (buffer == NULL || !GST_BUFFER_PTS_IS_VALID(buffer)) { + fail_run(run, "capture input has no presentation timestamp"); + return GST_PAD_PROBE_DROP; + } + GstClockTime timestamp = GST_BUFFER_PTS(buffer); + if (GST_CLOCK_TIME_IS_VALID(run->last_input_timestamp) && + timestamp <= run->last_input_timestamp) return GST_PAD_PROBE_DROP; + run->last_input_timestamp = timestamp; + GstClockTime duration = GST_BUFFER_DURATION_IS_VALID(buffer) + ? GST_BUFFER_DURATION(buffer) + : GST_SECOND / run->profile.frame_rate; + if (!write_frame(5, 0, 0, 0, 0, timestamp / 100, MAX(1, duration / 100), NULL, 0)) { + fail_run(run, "native media output closed"); + return GST_PAD_PROBE_DROP; + } + return GST_PAD_PROBE_OK; +} + +static GstPadProbeReturn output_input_frame(GstPad *pad, GstPadProbeInfo *info, + gpointer data) { + VideoOutput *output = data; + CaptureRun *run = output->run; + GstBuffer *buffer = GST_PAD_PROBE_INFO_BUFFER(info); + if (buffer == NULL || !GST_BUFFER_PTS_IS_VALID(buffer)) return GST_PAD_PROBE_DROP; + GstClockTime timestamp = GST_BUFFER_PTS(buffer); + g_mutex_lock(&run->lock); + gboolean enabled = output->enabled; + gboolean force = enabled && output->key_requested != output->key_sent; + guint64 request = output->key_requested; + if (enabled && !GST_CLOCK_TIME_IS_VALID(output->activation_timestamp)) { + output->activation_timestamp = timestamp; + } + if (force) { + output->key_sent = request; + output->key_timestamp = timestamp; + } + g_mutex_unlock(&run->lock); + if (!enabled) return GST_PAD_PROBE_DROP; + if (force) { + gboolean sent = gst_pad_push_event(pad, + gst_video_event_new_downstream_force_key_unit( + timestamp, GST_CLOCK_TIME_NONE, GST_CLOCK_TIME_NONE, TRUE, + (guint)request)); + if (!sent) { + g_mutex_lock(&run->lock); + if (output->key_sent == request) output->key_sent = output->key_acknowledged; + g_mutex_unlock(&run->lock); } } - g_atomic_int_set(&run->stopping, TRUE); - g_main_context_invoke(NULL, quit_loop, run); - return NULL; + return GST_PAD_PROBE_OK; } static gboolean bus_message(GstBus *bus, GstMessage *message, gpointer data) { @@ -523,6 +837,12 @@ static gboolean bus_message(GstBus *bus, GstMessage *message, gpointer data) { GError *error = NULL; char *debug = NULL; gst_message_parse_error(message, &error, &debug); + for (guint index = 0; index < run->output_count; ++index) { + if (GST_MESSAGE_SRC(message) == GST_OBJECT(run->outputs[index].encoder)) { + fail_output(&run->outputs[index], "hardware encoder output failed"); + break; + } + } fail_run(run, error == NULL ? "GStreamer pipeline failed" : error->message); g_clear_error(&error); g_free(debug); @@ -534,7 +854,8 @@ static gboolean bus_message(GstBus *bus, GstMessage *message, gpointer data) { } static GstFlowReturn video_sample(GstAppSink *sink, gpointer data) { - CaptureRun *run = data; + VideoOutput *output = data; + CaptureRun *run = output->run; GstSample *sample = gst_app_sink_pull_sample(sink); if (sample == NULL) return GST_FLOW_EOS; GstBuffer *buffer = gst_sample_get_buffer(sample); @@ -543,30 +864,44 @@ static GstFlowReturn video_sample(GstAppSink *sink, gpointer data) { mapped.size == 0 || mapped.size > kMaxPayloadBytes) { if (buffer != NULL && mapped.data != NULL) gst_buffer_unmap(buffer, &mapped); gst_sample_unref(sample); - fail_run(run, "GStreamer returned an invalid H.264 access unit"); + fail_output(output, "GStreamer returned an invalid H.264 access unit"); return GST_FLOW_ERROR; } gboolean key_frame = !GST_BUFFER_FLAG_IS_SET(buffer, GST_BUFFER_FLAG_DELTA_UNIT); NalSummary nal = inspect_h264(mapped.data, mapped.size); - if (!run->active && - (!key_frame || !nal.sps || !nal.pps || !nal.idr || - nal.profile_level_id[0] == '\0')) { + if (!GST_BUFFER_PTS_IS_VALID(buffer)) { + gst_buffer_unmap(buffer, &mapped); + gst_sample_unref(sample); + fail_output(output, "encoded output lost the source presentation timestamp"); + return GST_FLOW_ERROR; + } + GstClockTime timestamp = GST_BUFFER_PTS(buffer); + gboolean recovery = key_frame && nal.sps && nal.pps && nal.idr && + nal.profile_level_id[0] != '\0'; + g_mutex_lock(&run->lock); + gboolean current = output->enabled && + GST_CLOCK_TIME_IS_VALID(output->activation_timestamp) && + timestamp >= output->activation_timestamp; + if (current && GST_CLOCK_TIME_IS_VALID(output->key_timestamp) && + timestamp >= output->key_timestamp) { + if (recovery) output->key_acknowledged = output->key_sent; + else output->key_sent = output->key_acknowledged; + } + if (current && recovery) output->decodable = TRUE; + gboolean deliver = current && output->decodable; + gboolean publish_active = deliver && !run->active && + output->layer == run->original_output; + if (publish_active) run->active = TRUE; + g_mutex_unlock(&run->lock); + if (!deliver) { gst_buffer_unmap(buffer, &mapped); gst_sample_unref(sample); - g_main_context_invoke(NULL, force_key_frame, run); return GST_FLOW_OK; } - guint64 duration = GST_BUFFER_DURATION_IS_VALID(buffer) - ? GST_BUFFER_DURATION(buffer) / 100 - : 10000000ULL / run->profile.frame_rate; - guint64 timestamp = GST_BUFFER_PTS_IS_VALID(buffer) - ? GST_BUFFER_PTS(buffer) / 100 - : run->last_timestamp + duration; - if (run->last_timestamp != 0 && timestamp <= run->last_timestamp) { - timestamp = run->last_timestamp + duration; - } - run->last_timestamp = timestamp; - gboolean written = write_frame(2, key_frame ? 1 : 0, timestamp, duration, + gboolean written = write_frame(2, recovery ? 1 : 0, (guint8)output->layer, + output->profile.width, output->profile.height, + timestamp / 100, + 10000000ULL / output->profile.frame_rate, mapped.data, mapped.size); gst_buffer_unmap(buffer, &mapped); gst_sample_unref(sample); @@ -574,18 +909,20 @@ static GstFlowReturn video_sample(GstAppSink *sink, gpointer data) { fail_run(run, "native media output closed"); return GST_FLOW_ERROR; } - if (!run->active) { + if (publish_active) { char *token = json_string(run->restore_token); + char *profiles = output_profiles_json(run); char *status = g_strdup_printf( "{\"state\":\"active\",\"codec\":\"h264\",\"hardwareOnly\":true," "\"profileLevelId\":\"%s\",\"width\":%u,\"height\":%u," - "\"fps\":%u,\"restoreToken\":%s}", + "\"fps\":%u,\"restoreToken\":%s,\"outputs\":%s}", nal.profile_level_id, run->profile.width, run->profile.height, - run->profile.frame_rate, token); + run->profile.frame_rate, token, profiles); g_free(token); - run->active = write_status(status); + g_free(profiles); + gboolean active_written = write_status(status); g_free(status); - if (!run->active) { + if (!active_written) { fail_run(run, "native media output closed"); return GST_FLOW_ERROR; } @@ -620,7 +957,7 @@ static GstFlowReturn audio_sample(GstAppSink *sink, gpointer data) { gst_buffer_unmap(buffer, &mapped); gst_sample_unref(sample); while (run->audio->len >= kAudioFrameBytes) { - if (!write_frame(1, 0, run->audio_timestamp, 200000, + if (!write_frame(1, 0, 0, 0, 0, run->audio_timestamp, 200000, run->audio->data, kAudioFrameBytes)) { fail_run(run, "native audio output closed"); return GST_FLOW_ERROR; @@ -631,27 +968,34 @@ static GstFlowReturn audio_sample(GstAppSink *sink, gpointer data) { return GST_FLOW_OK; } -static int run_pipeline(CaptureRun *run, GCallback sample_callback, - const char *starting_status) { +static int run_pipeline(CaptureRun *run, const char *starting_status) { + run->input_payload = g_byte_array_new(); GstBus *bus = gst_element_get_bus(run->pipeline); guint bus_watch = gst_bus_add_watch(bus, bus_message, run); gst_object_unref(bus); - GstElement *output = gst_bin_get_by_name(GST_BIN(run->pipeline), "output"); - if (output == NULL) return 2; - g_signal_connect(output, "new-sample", sample_callback, run); - gst_object_unref(output); - if (starting_status != NULL && !write_status(starting_status)) return 2; + guint control_watch = g_unix_fd_add(STDIN_FILENO, G_IO_IN | G_IO_HUP | G_IO_ERR, + control_input, run); + if (starting_status != NULL && !write_status(starting_status)) { + fail_run(run, "native media output closed"); + goto stopped; + } GstStateChangeReturn state = gst_element_set_state(run->pipeline, GST_STATE_PLAYING); - if (state == GST_STATE_CHANGE_FAILURE) return 2; + if (state == GST_STATE_CHANGE_FAILURE) { + fail_run(run, "GStreamer pipeline failed to start"); + goto stopped; + } if (gst_element_get_state(run->pipeline, NULL, NULL, 5 * GST_SECOND) == GST_STATE_CHANGE_FAILURE) { - return 2; + fail_run(run, "GStreamer pipeline failed to start"); + goto stopped; } - g_thread_unref(g_thread_new("screener-control", control_input, run)); g_main_loop_run(run->loop); +stopped: + g_source_remove(control_watch); gst_element_set_state(run->pipeline, GST_STATE_NULL); g_source_remove(bus_watch); + g_byte_array_unref(run->input_payload); g_mutex_lock(&run->lock); char *failure = g_strdup(run->failure); g_mutex_unlock(&run->lock); @@ -664,18 +1008,22 @@ static int run_pipeline(CaptureRun *run, GCallback sample_callback, } static int capture_video(int count, char **values) { - VideoProfile profile = {0}; + CaptureRun run = {.last_input_timestamp = GST_CLOCK_TIME_NONE}; + gboolean encoded = count > 1 && strcmp(values[1], "--encoded-video") == 0; guint encoder_index = 0; - if (!parse_profile(count, values, &profile, &encoder_index)) return 2; + if (!parse_profile(count, values, &run, &encoder_index)) return 2; GPtrArray *encoders = hardware_encoders(); - if (!video_stack_available() || encoder_index >= encoders->len) { + gboolean input_available = encoded + ? has_factory("appsrc") && has_factory("h264parse") && has_factory("decodebin") + : video_stack_available(); + if (!input_available || encoder_index >= encoders->len) { g_ptr_array_unref(encoders); return 2; } const EncoderInfo *encoder_info = g_ptr_array_index(encoders, encoder_index); GError *error = NULL; ScreenerPortalCapture portal = {.pipewire_fd = -1}; - if (!screener_portal_capture_open( + if (!encoded && !screener_portal_capture_open( g_getenv("SCREENER_XDP_RESTORE_TOKEN"), &portal, &error)) { fprintf(stderr, "Screener capture unavailable: %s\n", error == NULL ? "screen selection failed" : error->message); @@ -684,19 +1032,36 @@ static int capture_video(int count, char **values) { return 2; } - char *pipeline_text = g_strdup_printf( + GString *pipeline_text = g_string_new(NULL); + if (encoded) { + g_string_append(pipeline_text, + "appsrc name=source is-live=true format=time do-timestamp=false block=true " + "max-buffers=1 max-bytes=1048576 max-time=0 " + "caps=video/x-h264,stream-format=byte-stream,alignment=au ! " + "h264parse ! decodebin ! video/x-raw ! tee name=frames "); + } else g_string_append_printf(pipeline_text, "pipewiresrc name=source fd=%d target-object=%s do-timestamp=true ! " - "queue max-size-buffers=2 max-size-bytes=0 max-size-time=0 " - "leaky=downstream ! videoconvert ! videoscale ! videorate ! " - "video/x-raw,format=NV12,width=%u,height=%u,framerate=%u/1 ! " - "%s name=encoder ! video/x-h264,profile=constrained-baseline ! " - "h264parse config-interval=-1 ! " - "video/x-h264,stream-format=byte-stream,alignment=au ! " - "appsink name=output emit-signals=true sync=false max-buffers=2 drop=true", - portal.pipewire_fd, portal.target_object, profile.width, profile.height, - profile.frame_rate, encoder_info->factory); - GstElement *pipeline = gst_parse_launch(pipeline_text, &error); - g_free(pipeline_text); + "videorate drop-only=true ! video/x-raw,framerate=%u/1 ! tee name=frames ", + portal.pipewire_fd, portal.target_object, run.profile.frame_rate); + for (guint index = 0; index < run.output_count; ++index) { + const VideoProfile *profile = &run.outputs[index].profile; + g_string_append_printf(pipeline_text, + "frames. ! valve name=gate%u drop=%s ! ( " + "queue name=raw%u max-size-buffers=1 max-size-bytes=0 " + "max-size-time=0 leaky=downstream ! videorate drop-only=true ! " + "video/x-raw,framerate=%u/1 ! videoconvert ! videoscale ! " + "video/x-raw,format=NV12,width=%u,height=%u ! " + "%s name=encoder%u ! video/x-h264,profile=constrained-baseline ! " + "h264parse config-interval=-1 ! " + "video/x-h264,stream-format=byte-stream,alignment=au ! " + "appsink name=output%u emit-signals=true sync=false async=false " + "max-buffers=1 drop=false ) ", + index, run.outputs[index].enabled ? "false" : "true", + index, profile->frame_rate, profile->width, profile->height, encoder_info->factory, + index, index); + } + GstElement *pipeline = gst_parse_launch(pipeline_text->str, &error); + g_string_free(pipeline_text, TRUE); if (pipeline == NULL || error != NULL) { fprintf(stderr, "Screener capture unavailable: %s\n", error == NULL ? "video pipeline failed" : error->message); @@ -706,48 +1071,85 @@ static int capture_video(int count, char **values) { g_ptr_array_unref(encoders); return 2; } - CaptureRun run = { - .loop = g_main_loop_new(NULL, FALSE), - .pipeline = pipeline, - .profile = profile, - .encoder_info = encoder_info, - .restore_token = portal.restore_token, - }; + run.loop = g_main_loop_new(NULL, FALSE); + run.pipeline = pipeline; + run.encoder_info = encoder_info; + run.restore_token = portal.restore_token; g_mutex_init(&run.lock); - run.encoder = gst_bin_get_by_name(GST_BIN(pipeline), "encoder"); + int result = 2; GstElement *source = gst_bin_get_by_name(GST_BIN(pipeline), "source"); if (source != NULL) { - set_optional_boolean(G_OBJECT(source), "resend-last", TRUE); - set_numeric_property(G_OBJECT(source), "keepalive-time", - MAX(1, 1000 / profile.frame_rate)); - gst_object_unref(source); + if (encoded) run.encoded_source = source; + else { + set_optional_boolean(G_OBJECT(source), "resend-last", TRUE); + set_numeric_property(G_OBJECT(source), "keepalive-time", + MAX(1, 1000 / run.profile.frame_rate)); + gst_object_unref(source); + } } - if (run.encoder == NULL || !configure_encoder(&run)) { - fprintf(stderr, "Screener capture unavailable: hardware encoder is incompatible\n"); - if (run.encoder != NULL) gst_object_unref(run.encoder); - g_main_loop_unref(run.loop); - g_mutex_clear(&run.lock); - gst_object_unref(pipeline); - screener_portal_capture_close(&portal); - g_ptr_array_unref(encoders); - return 2; + GstElement *tee = gst_bin_get_by_name(GST_BIN(pipeline), "frames"); + GstPad *input = tee == NULL ? NULL : gst_element_get_static_pad(tee, "sink"); + if (tee != NULL) gst_object_unref(tee); + if (input == NULL) goto cleanup; + gst_pad_add_probe(input, GST_PAD_PROBE_TYPE_BUFFER, begin_input_frame, &run, NULL); + gst_object_unref(input); + for (guint index = 0; index < run.output_count; ++index) { + VideoOutput *output = &run.outputs[index]; + char *name = g_strdup_printf("encoder%u", index); + output->encoder = gst_bin_get_by_name(GST_BIN(pipeline), name); + g_free(name); + if (output->encoder == NULL || !configure_encoder(output)) { + fprintf(stderr, "Screener capture unavailable: hardware encoder is incompatible\n"); + write_frame(6, 0, (guint8)index, 0, 0, 0, 0, + (const guint8 *)"hardware encoder configuration failed", + strlen("hardware encoder configuration failed")); + goto cleanup; + } + output->branch = GST_ELEMENT(gst_object_get_parent(GST_OBJECT(output->encoder))); + name = g_strdup_printf("gate%u", index); + output->gate = gst_bin_get_by_name(GST_BIN(pipeline), name); + g_free(name); + if (output->branch == NULL || output->branch == pipeline || output->gate == NULL) goto cleanup; + gst_element_set_locked_state(output->branch, !output->enabled); + name = g_strdup_printf("output%u", index); + GstElement *sink = gst_bin_get_by_name(GST_BIN(pipeline), name); + g_free(name); + if (sink == NULL) goto cleanup; + g_signal_connect(sink, "new-sample", G_CALLBACK(video_sample), output); + gst_object_unref(sink); + name = g_strdup_printf("raw%u", index); + GstElement *queue = gst_bin_get_by_name(GST_BIN(pipeline), name); + g_free(name); + GstPad *raw = queue == NULL ? NULL : gst_element_get_static_pad(queue, "src"); + if (queue != NULL) gst_object_unref(queue); + if (raw == NULL) goto cleanup; + gst_pad_add_probe(raw, GST_PAD_PROBE_TYPE_BUFFER, output_input_frame, output, NULL); + gst_object_unref(raw); } char *adapter_name = json_string("GStreamer hardware H.264"); char *encoder_name = json_string(encoder_info->name); char *encoder_identity = json_string(encoder_info->factory); + char *profiles = output_profiles_json(&run); char *starting = g_strdup_printf( "{\"state\":\"starting\",\"codec\":\"h264\",\"hardwareOnly\":true," "\"adapterIndex\":0,\"adapterName\":%s," "\"adapterIdentity\":\"gstreamer-hardware-h264\"," "\"encoderIndex\":%u,\"encoderName\":%s," - "\"encoderIdentity\":%s}", - adapter_name, encoder_index, encoder_name, encoder_identity); + "\"encoderIdentity\":%s,\"outputs\":%s}", + adapter_name, encoder_index, encoder_name, encoder_identity, profiles); g_free(adapter_name); g_free(encoder_name); g_free(encoder_identity); - int result = run_pipeline(&run, G_CALLBACK(video_sample), starting); + g_free(profiles); + result = run_pipeline(&run, starting); g_free(starting); - gst_object_unref(run.encoder); +cleanup: + if (run.encoded_source != NULL) gst_object_unref(run.encoded_source); + for (guint index = 0; index < run.output_count; ++index) { + if (run.outputs[index].encoder != NULL) gst_object_unref(run.outputs[index].encoder); + if (run.outputs[index].branch != NULL) gst_object_unref(run.outputs[index].branch); + if (run.outputs[index].gate != NULL) gst_object_unref(run.outputs[index].gate); + } g_free(run.failure); g_main_loop_unref(run.loop); g_mutex_clear(&run.lock); @@ -784,7 +1186,12 @@ static int capture_audio(int count, char **values) { .audio = g_byte_array_new(), }; g_mutex_init(&run.lock); - int result = run_pipeline(&run, G_CALLBACK(audio_sample), NULL); + GstElement *output = gst_bin_get_by_name(GST_BIN(pipeline), "output"); + if (output != NULL) { + g_signal_connect(output, "new-sample", G_CALLBACK(audio_sample), &run); + gst_object_unref(output); + } + int result = output == NULL ? 2 : run_pipeline(&run, NULL); g_byte_array_unref(run.audio); g_free(run.failure); g_main_loop_unref(run.loop); @@ -797,7 +1204,7 @@ int main(int argc, char **argv) { gst_init(&argc, &argv); if (argc == 2 && strcmp(argv[1], "--probe") == 0) return write_probe(); if (argc == 2 && strcmp(argv[1], "--list") == 0) return write_sources(); - if (argc > 1 && strcmp(argv[1], "--capture-video") == 0) { + if (argc > 1 && (strcmp(argv[1], "--capture-video") == 0 || strcmp(argv[1], "--encoded-video") == 0)) { return capture_video(argc, argv); } if (argc > 1 && strcmp(argv[1], "--capture-audio") == 0) { diff --git a/native/capture/linux/main.test.c b/native/capture/linux/main.test.c new file mode 100644 index 0000000..b42aaf4 --- /dev/null +++ b/native/capture/linux/main.test.c @@ -0,0 +1,118 @@ +// Build with portal.c and the same pkg-config flags as build.sh; no capture or codec is used. +#define main capture_main +#include "main.c" +#undef main + +static void check_slot_profiles(void) { + char *arguments[23 + 7 * 5] = { + "capture", "--capture-video", "picker", "1", "0", "0", + "--adapter-index", "0", "--mft-index", "0", "--width", "1280", + "--height", "720", "--fps", "30", "--bitrate", "3000000", + "--preference", "balanced", "--codec", "h264", "--protocol-v7"}; + for (guint index = 0; index < 7; ++index) { + guint offset = 23 + index * 5; + arguments[offset] = "--output"; + arguments[offset + 1] = index == 1 ? "1280" : "854"; + arguments[offset + 2] = index == 1 ? "720" : "480"; + arguments[offset + 3] = "30"; + arguments[offset + 4] = index == 1 ? "3000000" : "2000000"; + } + CaptureRun run = {0}; + guint encoder = 0; + g_assert_true(parse_profile(53, arguments, &run, &encoder)); + g_assert_cmpuint(run.output_count, ==, 6); + g_assert_cmpuint(run.original_output, ==, 1); + for (guint index = 0; index < run.output_count; ++index) { + g_assert_cmpint(run.outputs[index].enabled, ==, index < 2); + } + g_assert_false(parse_profile(58, arguments, &run, &encoder)); + arguments[23 + 5 * 5 + 1] = "1920"; + g_assert_false(parse_profile(53, arguments, &run, &encoder)); + arguments[23 + 5 * 5 + 1] = "854"; + arguments[23 + 5 + 1] = "854"; + g_assert_false(parse_profile(53, arguments, &run, &encoder)); + arguments[23 + 5 + 1] = "1280"; + + char *encoded[11 + 6 * 5] = { + "capture", "--encoded-video", "--codec", "h264", "--adapter-index", "0", + "--mft-index", "0", "--preference", "balanced", "--protocol-v7"}; + memcpy(encoded + 11, arguments + 23, 6 * 5 * sizeof(char *)); + run = (CaptureRun){0}; + g_assert_true(parse_profile(G_N_ELEMENTS(encoded), encoded, &run, &encoder)); + g_assert_cmpuint(run.profile.width, ==, 1280); + g_assert_cmpuint(run.profile.height, ==, 720); + g_assert_cmpuint(run.original_output, ==, 0); + for (guint index = 0; index < run.output_count; ++index) { + g_assert_false(run.outputs[index].enabled); + } +} + +static GstState branch_state(VideoOutput *output) { + GstState state = GST_STATE_VOID_PENDING; + g_assert_cmpint(gst_element_get_state(output->branch, &state, NULL, GST_SECOND), + !=, GST_STATE_CHANGE_FAILURE); + return state; +} + +static void check_slot_retirement(void) { + CaptureRun run = {.output_count = 6}; + g_mutex_init(&run.lock); + GString *description = g_string_new("fakesrc is-live=true num-buffers=8 ! tee name=frames "); + for (guint index = 0; index < run.output_count; ++index) { + g_string_append_printf(description, + "frames. ! valve name=gate%u drop=true ! ( identity name=encoder%u ! " + "fakesink sync=false async=false ) ", index, index); + } + GError *error = NULL; + run.pipeline = gst_parse_launch(description->str, &error); + g_string_free(description, TRUE); + g_assert_no_error(error); + g_assert_nonnull(run.pipeline); + for (guint index = 0; index < run.output_count; ++index) { + VideoOutput *output = &run.outputs[index]; + output->run = &run; + output->profile.bit_rate = 2000000; + char *name = g_strdup_printf("encoder%u", index); + output->encoder = gst_bin_get_by_name(GST_BIN(run.pipeline), name); + g_free(name); + g_assert_nonnull(output->encoder); + output->branch = GST_ELEMENT(gst_object_get_parent(GST_OBJECT(output->encoder))); + g_assert_true(output->branch != run.pipeline); + name = g_strdup_printf("gate%u", index); + output->gate = gst_bin_get_by_name(GST_BIN(run.pipeline), name); + g_free(name); + g_assert_nonnull(output->gate); + gst_element_set_locked_state(output->branch, TRUE); + } + g_assert_cmpint(gst_element_set_state(run.pipeline, GST_STATE_PLAYING), !=, GST_STATE_CHANGE_FAILURE); + g_assert_true(apply_control(&run, "A 1 1")); + g_assert_true(apply_control(&run, "A 5 1")); + g_assert_cmpint(branch_state(&run.outputs[1]), ==, GST_STATE_PLAYING); + g_assert_cmpint(branch_state(&run.outputs[5]), ==, GST_STATE_PLAYING); + g_assert_cmpint(branch_state(&run.outputs[2]), ==, GST_STATE_NULL); + g_assert_true(apply_control(&run, "A 1 0")); + g_assert_cmpint(branch_state(&run.outputs[1]), ==, GST_STATE_NULL); + g_assert_cmpint(branch_state(&run.outputs[5]), ==, GST_STATE_PLAYING); + g_assert_true(apply_control(&run, "A 1 1")); + g_assert_cmpint(branch_state(&run.outputs[1]), ==, GST_STATE_PLAYING); + g_assert_true(apply_control(&run, "K 5")); + g_assert_false(apply_control(&run, "A 2")); + g_assert_false(apply_control(&run, "A 6 1")); + g_assert_false(apply_control(&run, "A 0 2")); + g_assert_false(apply_control(&run, "B 5 999")); + gst_element_set_state(run.pipeline, GST_STATE_NULL); + for (guint index = 0; index < run.output_count; ++index) { + gst_object_unref(run.outputs[index].encoder); + gst_object_unref(run.outputs[index].branch); + gst_object_unref(run.outputs[index].gate); + } + gst_object_unref(run.pipeline); + g_mutex_clear(&run.lock); +} + +int main(int argc, char **argv) { + gst_init(&argc, &argv); + check_slot_profiles(); + check_slot_retirement(); + return 0; +} diff --git a/native/capture/windows/README.md b/native/capture/windows/README.md index c7b6b37..1a44906 100644 --- a/native/capture/windows/README.md +++ b/native/capture/windows/README.md @@ -27,8 +27,8 @@ codec remains fixed across profile and source changes. VP8 reads the existing NV12 surface through one staging texture and uses the same encoded-frame boundary. The process has no network fallback. The Client consumes the selected process or system-audio stream through its native media -edge when the capability probe reports support. Build it outside the repository -for a bounded capability run: +edge when the capability probe reports support. Build to the stable project +`build/client-check` directory for a bounded capability run: ```powershell npm run check:client-capture @@ -38,8 +38,17 @@ The implementation follows Microsoft's MIT-licensed reference samples and official API contracts without copying their WIL framework. The retained MF fixture compiles the same encoder source with `SCREENER_H264_FIXTURE`; there is not a second product MFT implementation. -The build caches pinned libvpx, NASM and make inputs under `build/client-check`; -only statically linked VP8 code and its notices enter the package. +Shared local outputs use stock WebRTC source/encoder adaptation. H264 keeps +native NV12 surfaces, while WebRTC VP8 reads back into I420. The existing capture +and output mailboxes retain source/generation ownership; adapted sizes and +timestamps cross the same media envelope. Hardware low-latency mode stays on; +the old fixed one-frame VBV override is removed so later bitrate increases are +not trapped by the original buffer limit. + +The build caches the pinned WebRTC SDK and matching official MSVC linker/runtime +libraries under `build/encoder-pool`. The SDK supplies libvpx, replacing the +separate source build. Only statically linked code and required notices enter +the package, not the SDK/toolchain or another runtime service. - - diff --git a/native/capture/windows/adaptive_encoder.cpp b/native/capture/windows/adaptive_encoder.cpp new file mode 100644 index 0000000..f848379 --- /dev/null +++ b/native/capture/windows/adaptive_encoder.cpp @@ -0,0 +1,600 @@ +#include "adaptive_encoder.h" +#include "capture_output.h" + +#include +#include +#include +#include +#include +#include +#include + +#include "api/environment/environment_factory.h" +#include "api/make_ref_counted.h" +#include "api/task_queue/task_queue_base.h" +#include "api/task_queue/task_queue_factory.h" +#include "api/video/builtin_video_bitrate_allocator_factory.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_adapter.h" +#include "api/video/video_frame.h" +#include "api/video/video_stream_encoder_settings.h" +#include "api/video_codecs/builtin_video_encoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "call/video_send_stream.h" +#include "libyuv/convert.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "system_wrappers/include/clock.h" +#include "video/adaptation/overuse_frame_detector.h" +#include "video/frame_cadence_adapter.h" +#include "video/send_statistics_proxy.h" +#include "video/video_stream_encoder.h" + +namespace screener::capture::windows { +namespace { + +template +void Sync(webrtc::TaskQueueBase* queue, F fn) { + std::promise completed; + auto result = completed.get_future(); + queue->PostTask([completed = std::move(completed), fn = std::move(fn)]() mutable { + try { + fn(); + completed.set_value(); + } catch (...) { + completed.set_exception(std::current_exception()); + } + }); + result.get(); +} + +// Library callbacks cannot throw through WebRTC's task queues. +class Failure final { + public: + void Capture() noexcept { + std::lock_guard lock(mutex_); + if (!failure_) failure_ = std::current_exception(); + } + void Set(const char* detail) noexcept { + try { + throw GateFailure("adaptive-encoder", detail); + } catch (...) { + Capture(); + } + } + void Rethrow() { + std::exception_ptr failure; + { + std::lock_guard lock(mutex_); + failure = failure_; + } + if (failure) { + try { + std::rethrow_exception(failure); + } catch (const GateFailure&) { + throw; + } catch (const std::exception& error) { + throw GateFailure("adaptive-encoder", error.what()); + } catch (...) { + throw GateFailure("adaptive-encoder", "WebRTC encoder callback failed"); + } + } + } + + private: + std::mutex mutex_; + std::exception_ptr failure_; +}; + +class Readback final { + public: + Readback(ID3D11Device* device, Failure& failure) : device_(device), failure_(failure) { + device_->GetImmediateContext(&context_); + } + + webrtc::scoped_refptr Convert(ID3D11Texture2D* texture) { + try { + D3D11_TEXTURE2D_DESC description{}; + texture->GetDesc(&description); + if (!staging_ || width_ != description.Width || height_ != description.Height) { + staging_.Reset(); + description.Usage = D3D11_USAGE_STAGING; + description.BindFlags = 0; + description.CPUAccessFlags = D3D11_CPU_ACCESS_READ; + description.MiscFlags = 0; + Check(device_->CreateTexture2D(&description, nullptr, &staging_), + "adaptive-vp8-staging"); + width_ = description.Width; + height_ = description.Height; + } + auto pixels = webrtc::I420Buffer::Create(static_cast(width_), + static_cast(height_)); + context_->CopyResource(staging_.Get(), texture); + D3D11_MAPPED_SUBRESOURCE mapped{}; + Check(context_->Map(staging_.Get(), 0, D3D11_MAP_READ, 0, &mapped), + "adaptive-vp8-readback"); + const auto* y = static_cast(mapped.pData); + const int result = libyuv::NV12ToI420( + y, static_cast(mapped.RowPitch), y + mapped.RowPitch * height_, + static_cast(mapped.RowPitch), pixels->MutableDataY(), pixels->StrideY(), + pixels->MutableDataU(), pixels->StrideU(), pixels->MutableDataV(), + pixels->StrideV(), static_cast(width_), static_cast(height_)); + context_->Unmap(staging_.Get(), 0); + if (result != 0) Fail("adaptive-vp8-convert", "NV12 conversion failed"); + return pixels; + } catch (...) { + failure_.Capture(); + return nullptr; + } + } + + private: + ComPtr device_; + ComPtr context_; + ComPtr staging_; + Failure& failure_; + UINT32 width_ = 0, height_ = 0; +}; + +class NativeVideoFrameBuffer : public webrtc::VideoFrameBuffer { + public: + NativeVideoFrameBuffer(ComPtr texture, UINT32 width, UINT32 height, + UINT64 timestamp100ns, Readback& readback) + : texture_(std::move(texture)), width_(width), height_(height), + timestamp100ns_(timestamp100ns), readback_(readback) {} + Type type() const override { return Type::kNative; } + int width() const override { return static_cast(width_); } + int height() const override { return static_cast(height_); } + webrtc::scoped_refptr ToI420() override { + return readback_.Convert(texture_.Get()); + } + ID3D11Texture2D* texture() const { return texture_.Get(); } + UINT64 timestamp100ns() const { return timestamp100ns_; } + + private: + ComPtr texture_; + const UINT32 width_, height_; + const UINT64 timestamp100ns_; + Readback& readback_; +}; + +class HardwareEncoder final : public webrtc::VideoEncoder { + public: + HardwareEncoder(const webrtc::Environment& env, VideoProfile ceiling, + AdaptiveEncoder::Factory create, + std::unique_ptr initial, + Failure& failure) + : env_(env), ceiling_(ceiling), create_(std::move(create)), + initial_(std::move(initial)), failure_(failure) { + Check(CoInitializeEx(nullptr, COINIT_MULTITHREADED), "adaptive-h264-apartment"); + } + ~HardwareEncoder() override { + initial_.reset(); + Release(); + CoUninitialize(); + } + + int32_t InitEncode(const webrtc::VideoCodec* config, const Settings&) override { + Release(); + try { + if (!config || config->codecType != webrtc::kVideoCodecH264 || + config->width == 0 || config->height == 0 || (config->width & 1) || + (config->height & 1) || config->width > ceiling_.width || + config->height > ceiling_.height || config->maxFramerate == 0 || + config->maxFramerate > ceiling_.frame_rate || config->startBitrate == 0 || + config->numberOfSimulcastStreams > 1 || config->H264().numberOfTemporalLayers > 1) { + Fail("adaptive-h264-config", "WebRTC requested an unsupported H264 configuration"); + } + profile_ = ceiling_; + profile_.width = config->width; + profile_.height = config->height; + profile_.frame_rate = config->maxFramerate; + profile_.bit_rate = config->startBitrate * 1000; + if (initial_ && profile_.width == ceiling_.width && + profile_.height == ceiling_.height && profile_.frame_rate == ceiling_.frame_rate) { + encoder_ = std::move(initial_); + encoder_->SetBitrate(profile_.bit_rate); + } else { + initial_.reset(); + encoder_ = create_(profile_); + } + if (!encoder_ || encoder_->kind != OutputKind::h264) + Fail("adaptive-h264-create", "The native H264 encoder is unavailable"); + applied_bitrate_ = profile_.bit_rate; + return WEBRTC_VIDEO_CODEC_OK; + } catch (...) { + failure_.Capture(); + Release(); + return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; + } + } + + int32_t RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback* callback) override { + callback_ = callback; + return WEBRTC_VIDEO_CODEC_OK; + } + int32_t Release() override { + encoder_.reset(); + paused_ = false; + key_frame_pending_ = true; + applied_bitrate_ = 0; + return WEBRTC_VIDEO_CODEC_OK; + } + + void SetRates(const RateControlParameters& rates) override { + if (!encoder_) return; + try { + const UINT32 bitrate = rates.bitrate.get_sum_bps(); + if (bitrate != rates.bitrate.GetBitrate(0, 0) || + !std::isfinite(rates.framerate_fps)) + Fail("adaptive-h264-rates", "WebRTC supplied unsupported H264 rates"); + if (bitrate == 0) { + paused_ = true; + return; + } + const double fps = rates.framerate_fps > 0 ? rates.framerate_fps : profile_.frame_rate; + // Chromium M152 MF compensates bitrate when requested FPS differs from configured FPS. + const double adjusted = static_cast(bitrate) * profile_.frame_rate / fps; + if (!std::isfinite(adjusted) || adjusted < 1 || + adjusted > std::numeric_limits::max()) + Fail("adaptive-h264-rates", "H264 rate compensation overflowed"); + const auto next = static_cast(adjusted); + if (next != applied_bitrate_) encoder_->SetBitrate(next); + applied_bitrate_ = next; + key_frame_pending_ = key_frame_pending_ || paused_; + paused_ = false; + } catch (...) { + failure_.Capture(); + } + } + + int32_t Encode(const webrtc::VideoFrame& frame, + const std::vector* requested) override { + try { + failure_.Rethrow(); + if (!encoder_ || !callback_) + Fail("adaptive-h264-encode", "The H264 encoder is not initialized"); + key_frame_pending_ = key_frame_pending_ || + (requested && std::find(requested->begin(), requested->end(), + webrtc::VideoFrameType::kVideoFrameKey) != requested->end()); + if (paused_ || (requested && !requested->empty() && + requested->front() == webrtc::VideoFrameType::kEmptyFrame)) { + callback_->OnFrameDropped(frame.rtp_timestamp(), 0, true); + return WEBRTC_VIDEO_CODEC_OK; + } + const auto* native = dynamic_cast(frame.video_frame_buffer().get()); + if (!native || frame.width() != static_cast(profile_.width) || + frame.height() != static_cast(profile_.height)) + Fail("adaptive-h264-input", "H264 requires the selected native NV12 surface"); + const auto began = env_.clock().TimeInMilliseconds(); + const auto access_unit = encoder_->Encode(native->texture(), native->timestamp100ns(), + key_frame_pending_); + key_frame_pending_ = false; + webrtc::EncodedImage image; + image.SetEncodedData(webrtc::EncodedImageBuffer::Create( + access_unit.bytes.data(), access_unit.bytes.size())); + image._encodedWidth = profile_.width; + image._encodedHeight = profile_.height; + image.SetRtpTimestamp(frame.rtp_timestamp()); + image.capture_time_ms_ = frame.timestamp_us() / 1000; + image.ntp_time_ms_ = frame.ntp_time_ms(); + image.SetPresentationTimestamp(frame.presentation_timestamp()); + image.rotation_ = frame.rotation(); + image.SetColorSpace(frame.color_space()); + image.SetVideoFrameTrackingId(frame.id()); + image.SetSimulcastIndex(0); + image.SetSpatialIndex(0); + image.SetTemporalIndex(0); + image.set_end_of_temporal_unit(true); + image.SetFrameType(access_unit.key_frame ? webrtc::VideoFrameType::kVideoFrameKey + : webrtc::VideoFrameType::kVideoFrameDelta); + image.SetEncodeTime(began, env_.clock().TimeInMilliseconds()); + webrtc::CodecSpecificInfo info; + info.codecType = webrtc::kVideoCodecH264; + info.codecSpecific.H264 = {}; + info.codecSpecific.H264.packetization_mode = webrtc::H264PacketizationMode::NonInterleaved; + info.codecSpecific.H264.idr_frame = access_unit.key_frame; + info.scalability_mode = webrtc::ScalabilityMode::kL1T1; + if (callback_->OnEncodedImage(image, &info).error != webrtc::EncodedImageCallback::Result::OK) + Fail("adaptive-h264-callback", "WebRTC rejected the encoded H264 frame"); + return WEBRTC_VIDEO_CODEC_OK; + } catch (...) { + failure_.Capture(); + return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; + } + } + + EncoderInfo GetEncoderInfo() const override { + EncoderInfo info; + info.implementation_name = "Windows Media Foundation H264"; + info.is_hardware_accelerated = true; + info.enable_cpu_overuse_detection = true; + info.has_trusted_rate_controller = false; + // VSE parses H264 slice QP; use upstream H264 quality-scaler thresholds. + info.is_qp_trusted = true; + info.scaling_settings = ScalingSettings(24, 37); + info.requested_resolution_alignment = 2; + info.supports_native_handle = true; + info.supports_simulcast = false; + info.preferred_pixel_formats = {webrtc::VideoFrameBuffer::Type::kNative}; + return info; + } + + private: + const webrtc::Environment env_; + const VideoProfile ceiling_; + AdaptiveEncoder::Factory create_; + std::unique_ptr initial_, encoder_; + Failure& failure_; + VideoProfile profile_; + webrtc::EncodedImageCallback* callback_ = nullptr; + UINT32 applied_bitrate_ = 0; + bool paused_ = false, key_frame_pending_ = true; +}; + +webrtc::DegradationPreference Preference(DegradationPreference preference) { + switch (preference) { + case DegradationPreference::resolution: return webrtc::DegradationPreference::MAINTAIN_RESOLUTION; + case DegradationPreference::framerate: return webrtc::DegradationPreference::MAINTAIN_FRAMERATE; + case DegradationPreference::balanced: return webrtc::DegradationPreference::BALANCED; + } + Fail("adaptive-preference", "Unknown degradation preference"); +} + +} // namespace + +class AdaptiveEncoder::Impl final : public webrtc::VideoSourceInterface, + public webrtc::VideoStreamEncoderInterface::EncoderSink, + public webrtc::VideoEncoderFactory { + public: + Impl(OutputKind kind, VideoProfile ceiling, ID3D11Device* device, + Factory create, std::unique_ptr initial) + : kind_(kind), ceiling_(ceiling), device_(device), create_(std::move(create)), + initial_(std::move(initial)), env_(webrtc::CreateEnvironment()), + readback_(device, failure_), builtin_(webrtc::CreateBuiltinVideoEncoderFactory()), + allocator_(webrtc::CreateBuiltinVideoBitrateAllocatorFactory()), + worker_(env_.task_queue_factory().CreateTaskQueue( + "CaptureAdaptation", webrtc::TaskQueueFactory::Priority::kNormal)) {} + + ~Impl() override { + Sync(worker_.get(), [&] { + if (encoder_) { + encoder_->Stop(); + encoder_.reset(); + encode_queue_ = nullptr; + } + }); + } + + void Start() { + Sync(worker_.get(), [&] { + const auto format = GetSupportedFormats().front(); + webrtc::VideoSendStream::Config send_config(nullptr); + send_config.rtp.ssrcs = {1}; + send_config.rtp.payload_name = format.name; + send_config.rtp.payload_type = 96; + stats_ = std::make_unique( + &env_.clock(), send_config, webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo, + env_.field_trials()); + webrtc::VideoStreamEncoderSettings settings(webrtc::VideoEncoder::Capabilities(false)); + settings.encoder_factory = this; + settings.bitrate_allocator_factory = allocator_.get(); + auto queue = env_.task_queue_factory().CreateTaskQueue( + "CaptureEncoder", webrtc::TaskQueueFactory::Priority::kNormal); + encode_queue_ = queue.get(); + auto cadence = webrtc::FrameCadenceAdapterInterface::Create( + &env_.clock(), queue.get(), nullptr, webrtc::TaskQueueBase::Current(), env_.field_trials()); + encoder_ = std::make_unique( + env_, static_cast(std::max(1u, std::thread::hardware_concurrency())), + stats_.get(), settings, std::make_unique(env_, stats_.get()), + std::move(cadence), std::move(queue), + webrtc::VideoStreamEncoder::BitrateAllocationCallbackType::kVideoBitrateAllocation, + nullptr, [this](std::optional, bool) { + failure_.Set("WebRTC could not continue with the selected encoder"); + }); + adapter_.OnOutputFormatRequest(std::nullopt, + static_cast(ceiling_.width * ceiling_.height), static_cast(ceiling_.frame_rate)); + encoder_->SetSink(this, false); + encoder_->SetSource(this, Preference(ceiling_.preference)); + encoder_->SetStartBitrate(static_cast(ceiling_.bit_rate)); + webrtc::VideoEncoderConfig config; + config.codec_type = kind_ == OutputKind::h264 ? webrtc::kVideoCodecH264 : webrtc::kVideoCodecVP8; + config.video_format = format; + config.content_type = webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo; + config.number_of_streams = 1; + config.max_bitrate_bps = static_cast(ceiling_.bit_rate); + config.frame_drop_enabled = true; + config.is_quality_scaling_allowed = true; + config.simulcast_layers.resize(1); + config.simulcast_layers[0].active = true; + config.simulcast_layers[0].max_framerate = static_cast(ceiling_.frame_rate); + config.simulcast_layers[0].max_bitrate_bps = static_cast(ceiling_.bit_rate); + config.simulcast_layers[0].num_temporal_layers = 1; + if (kind_ == OutputKind::vp8) { + auto vp8 = webrtc::VideoEncoder::GetDefaultVp8Settings(); + vp8.automaticResizeOn = true; + config.encoder_specific_settings = + webrtc::make_ref_counted(vp8); + } + encoder_->ConfigureEncoder(std::move(config), 1200, [this](webrtc::RTCError error) { + if (!error.ok()) failure_.Set("WebRTC encoder configuration failed"); + }); + }); + Sync(encode_queue_, [] {}); + failure_.Rethrow(); + } + + std::optional Encode(FrameProducer produce, UINT32 source_width, + UINT32 source_height, UINT64 timestamp100ns, bool key_frame, UINT32 bitrate) { + if (!produce || source_width == 0 || source_height == 0 || + source_width > ceiling_.width || source_height > ceiling_.height || + timestamp100ns > static_cast(std::numeric_limits::max())) + Fail("adaptive-input", "Invalid adaptive encoder input"); + failure_.Rethrow(); + Sync(worker_.get(), [&] { + const auto rate = webrtc::DataRate::BitsPerSec(std::min(bitrate, ceiling_.bit_rate)); + if (!bitrate_ || *bitrate_ != rate.bps()) { + encoder_->OnBitrateUpdated(rate, rate, 0, 0, 0); + bitrate_ = rate.bps(); + } + if (key_frame) encoder_->SendKeyFrame(); + if (!source_sink_) return; + if (!anchor100ns_) { + anchor100ns_ = timestamp100ns; + anchor_us_ = env_.clock().TimeInMicroseconds(); + anchor_ntp_ms_ = env_.clock().CurrentNtpInMilliseconds(); + } + if (timestamp100ns < *anchor100ns_) + Fail("adaptive-timestamp", "Capture timestamp moved before its output anchor"); + const auto elapsed = timestamp100ns - *anchor100ns_; + const INT64 capture_us = anchor_us_ + static_cast(elapsed / 10); + const INT64 ntp_ms = anchor_ntp_ms_ + static_cast(elapsed / 10'000); + int crop_width, crop_height, width, height; + if (!adapter_.AdaptFrameResolution(static_cast(source_width), + static_cast(source_height), capture_us * 1000, + &crop_width, &crop_height, &width, &height)) { + source_sink_->OnDiscardedFrame(); + return; + } + auto texture = produce(static_cast(width), static_cast(height)); + if (!texture) Fail("adaptive-surface", "The selected NV12 surface is unavailable"); + D3D11_TEXTURE2D_DESC description{}; + texture->GetDesc(&description); + ComPtr texture_device; + texture->GetDevice(&texture_device); + if (texture_device.Get() != device_.Get() || description.Format != DXGI_FORMAT_NV12 || + description.Width != static_cast(width) || + description.Height != static_cast(height)) + Fail("adaptive-surface", "The selected NV12 surface has a different device or size"); + { + std::lock_guard lock(output_mutex_); + timestamp100ns_ = timestamp100ns; + rtp_timestamp_ = 90u * static_cast(ntp_ms); + const double fps = std::min(ceiling_.frame_rate, adapter_.GetMaxFramerate()); + duration100ns_ = static_cast(10'000'000 / fps); + } + auto buffer = webrtc::make_ref_counted( + std::move(texture), static_cast(width), static_cast(height), + timestamp100ns, readback_); + source_sink_->OnFrame(webrtc::VideoFrame::Builder().set_video_frame_buffer(buffer) + .set_timestamp_us(capture_us).set_ntp_time_ms(ntp_ms).build()); + }); + // Both selected codecs complete synchronously; native frames cannot be retained by VSE. + Sync(encode_queue_, [] {}); + Sync(worker_.get(), [] {}); + failure_.Rethrow(); + std::lock_guard lock(output_mutex_); + return std::exchange(output_, std::nullopt); + } + + void AddOrUpdateSink(webrtc::VideoSinkInterface* sink, + const webrtc::VideoSinkWants& wants) override { + source_sink_ = sink; + adapter_.OnSinkWants(wants); + } + void RemoveSink(webrtc::VideoSinkInterface* sink) override { + if (source_sink_ == sink) source_sink_ = nullptr; + } + + std::vector GetSupportedFormats() const override { + if (kind_ == OutputKind::vp8) return {webrtc::SdpVideoFormat("VP8")}; + return {webrtc::SdpVideoFormat("H264", {{"profile-level-id", ceiling_.profile_level_id()}, + {"packetization-mode", "1"}, {"level-asymmetry-allowed", "1"}})}; + } + std::unique_ptr Create(const webrtc::Environment& env, + const webrtc::SdpVideoFormat& format) override { + try { + if (kind_ == OutputKind::vp8) { + initial_.reset(); + auto encoder = builtin_->Create(env, format); + if (!encoder) Fail("adaptive-vp8-create", "WebRTC VP8 encoder is unavailable"); + return encoder; + } + return std::make_unique(env, ceiling_, create_, std::move(initial_), failure_); + } catch (...) { + failure_.Capture(); + return nullptr; + } + } + + void OnEncoderConfigurationChanged(std::vector, bool, + webrtc::VideoEncoderConfig::ContentType, int) override {} + void OnBitrateAllocationUpdated(const webrtc::VideoBitrateAllocation&) override {} + void OnVideoLayersAllocationUpdated(webrtc::VideoLayersAllocation) override {} + void OnFrameDropped(uint32_t, int, bool) override {} + Result OnEncodedImage(const webrtc::EncodedImage& image, + const webrtc::CodecSpecificInfo*) override { + try { + std::lock_guard lock(output_mutex_); + if (output_) + Fail("adaptive-output-duplicate", "WebRTC returned more than one access unit for the current input"); + if (image.RtpTimestamp() != rtp_timestamp_) + Fail("adaptive-output-timestamp", "WebRTC returned an access unit from a different input"); + if (image.size() == 0 || image.size() > kMaxProductAccessUnitBytes) + Fail("adaptive-output-size", "Encoded access unit bytes=" + std::to_string(image.size())); + if (image._encodedWidth == 0 || image._encodedHeight == 0 || + image._encodedWidth > ceiling_.width || image._encodedHeight > ceiling_.height) + Fail("adaptive-output-dimensions", "WebRTC returned output outside its configured dimensions"); + AdaptiveAccessUnit output; + output.access_unit.timestamp100ns = timestamp100ns_; + output.access_unit.key_frame = image.IsKey(); + output.access_unit.bytes.assign(image.data(), image.data() + image.size()); + output.width = image._encodedWidth; + output.height = image._encodedHeight; + output.duration100ns = duration100ns_; + output_ = std::move(output); + return Result(Result::OK); + } catch (...) { + failure_.Capture(); + return Result(Result::ERROR_SEND_FAILED); + } + } + + private: + const OutputKind kind_; + const VideoProfile ceiling_; + ComPtr device_; + Factory create_; + std::unique_ptr initial_; + const webrtc::Environment env_; + Failure failure_; + Readback readback_; + std::unique_ptr builtin_; + std::unique_ptr allocator_; + std::unique_ptr stats_; + std::unique_ptr worker_; + std::unique_ptr encoder_; + webrtc::TaskQueueBase* encode_queue_ = nullptr; + webrtc::VideoSinkInterface* source_sink_ = nullptr; + webrtc::VideoAdapter adapter_{2}; + std::optional bitrate_; + std::optional anchor100ns_; + INT64 anchor_us_ = 0, anchor_ntp_ms_ = 0; + std::mutex output_mutex_; + std::optional output_; + UINT64 timestamp100ns_ = 0, duration100ns_ = 0; + UINT32 rtp_timestamp_ = 0; +}; + +AdaptiveEncoder::AdaptiveEncoder(OutputKind kind, VideoProfile ceiling, ID3D11Device* device, + Factory create, std::unique_ptr initial) { + if ((kind != OutputKind::vp8 && kind != OutputKind::h264) || !device || + (kind == OutputKind::h264 && !create) || + ceiling.width == 0 || ceiling.height == 0 || (ceiling.width & 1) || (ceiling.height & 1) || + static_cast(ceiling.width) * ceiling.height > std::numeric_limits::max() || + ceiling.frame_rate == 0 || ceiling.frame_rate > 1'000 || ceiling.bit_rate < 1'000 || + ceiling.bit_rate > static_cast(std::numeric_limits::max())) + Fail("adaptive-profile", "Invalid adaptive encoder profile"); + impl_ = std::make_unique(kind, ceiling, device, std::move(create), std::move(initial)); + impl_->Start(); +} + +AdaptiveEncoder::~AdaptiveEncoder() = default; + +std::optional AdaptiveEncoder::Encode(FrameProducer produce, + UINT32 source_width, UINT32 source_height, UINT64 timestamp100ns, + bool key_frame, UINT32 bitrate) { + return impl_->Encode(std::move(produce), source_width, source_height, timestamp100ns, + key_frame, bitrate); +} + +} // namespace screener::capture::windows diff --git a/native/capture/windows/adaptive_encoder.h b/native/capture/windows/adaptive_encoder.h new file mode 100644 index 0000000..ca4d1f6 --- /dev/null +++ b/native/capture/windows/adaptive_encoder.h @@ -0,0 +1,38 @@ +#pragma once + +#include "h264_encoder.h" + +#include +#include + +namespace screener::capture::windows { + +struct AdaptiveAccessUnit final { + EncodedAccessUnit access_unit; + UINT32 width = 0; + UINT32 height = 0; + UINT64 duration100ns = 0; +}; + +// One complete WebRTC encoding/adaptation pipeline for one shared output. +class AdaptiveEncoder final { + public: + using Factory = std::function(const VideoProfile&)>; + using FrameProducer = std::function(UINT32, UINT32)>; + + AdaptiveEncoder(OutputKind kind, VideoProfile ceiling, ID3D11Device* device, + Factory create, std::unique_ptr initial = nullptr); + ~AdaptiveEncoder(); + AdaptiveEncoder(const AdaptiveEncoder&) = delete; + AdaptiveEncoder& operator=(const AdaptiveEncoder&) = delete; + + std::optional Encode( + FrameProducer produce, UINT32 source_width, UINT32 source_height, + UINT64 timestamp100ns, bool key_frame, UINT32 bitrate); + + private: + class Impl; + std::unique_ptr impl_; +}; + +} // namespace screener::capture::windows diff --git a/native/capture/windows/build-libvpx.ps1 b/native/capture/windows/build-libvpx.ps1 deleted file mode 100644 index 3190bca..0000000 --- a/native/capture/windows/build-libvpx.ps1 +++ /dev/null @@ -1,68 +0,0 @@ -[CmdletBinding()] -param( - [Parameter(Mandatory = $true)] [string]$OutputDirectory, - [Parameter(Mandatory = $true)] [string]$VisualStudioDirectory -) - -Set-StrictMode -Version Latest -$ErrorActionPreference = 'Stop' -$ProgressPreference = 'SilentlyContinue' -$pinPath = Join-Path $PSScriptRoot 'libvpx-dependencies.json' -$pins = Get-Content -Raw $pinPath | ConvertFrom-Json -$outputPath = [IO.Path]::GetFullPath($OutputDirectory) -$sourcePath = Join-Path $outputPath ('libvpx-' + $pins.libvpx.commit) -$buildPath = Join-Path $outputPath 'build' -$libraryPath = Join-Path $buildPath 'x64\Release\vpxmt.lib' -$stampPath = Join-Path $buildPath 'build-stamp.txt' -$toolset = Get-ChildItem (Join-Path $VisualStudioDirectory 'VC\Tools\MSVC') -Directory | - Sort-Object Name -Descending | Select-Object -First 1 -$stamp = (Get-FileHash $pinPath).Hash + (Get-FileHash $PSCommandPath).Hash + $toolset.Name -if ((Test-Path $libraryPath) -and (Test-Path $stampPath) -and - (Get-Content -Raw $stampPath).Trim() -eq $stamp) { - return @{ Include = $sourcePath; Library = $libraryPath } -} - -New-Item -ItemType Directory -Path $outputPath,$buildPath -Force | Out-Null -function Get-PinnedArchive($pin, [string]$name) { - $archive = Join-Path $outputPath $name - if (!(Test-Path $archive) -or - (Get-FileHash $archive -Algorithm SHA256).Hash.ToLowerInvariant() -ne $pin.sha256) { - Invoke-WebRequest -UseBasicParsing -Uri $pin.url -OutFile $archive -TimeoutSec 120 - } - if ((Get-FileHash $archive -Algorithm SHA256).Hash.ToLowerInvariant() -ne $pin.sha256) { - throw "Native codec dependency digest mismatch: $name" - } - return $archive -} - -$sourceArchive = Get-PinnedArchive $pins.libvpx 'libvpx.tar.gz' -& tar -xf $sourceArchive -C $outputPath -if ($LASTEXITCODE -ne 0) { throw 'Could not unpack libvpx source.' } -$nasmArchive = Get-PinnedArchive $pins.nasm 'nasm.zip' -$nasmPath = Join-Path $outputPath ('nasm\nasm-' + $pins.nasm.version) -Expand-Archive -LiteralPath $nasmArchive -DestinationPath (Join-Path $outputPath 'nasm') -Force -$makeArchive = Get-PinnedArchive $pins.make 'make.pkg.tar.zst' -$makeRoot = Join-Path $outputPath 'make' -$makePath = Join-Path $makeRoot 'usr\bin' -New-Item -ItemType Directory -Path $makeRoot -Force | Out-Null -& tar -xf $makeArchive -C $makeRoot usr/bin/make.exe -if ($LASTEXITCODE -ne 0) { throw 'Could not unpack the libvpx build tool.' } - -$gitRoot = Split-Path -Parent (Split-Path -Parent (Get-Command git.exe).Source) -$bashPath = Join-Path $gitRoot 'bin\bash.exe' -if (!(Test-Path $bashPath)) { throw 'Git for Windows Bash is required to build libvpx.' } -$developerCommand = Join-Path $VisualStudioDirectory 'Common7\Tools\VsDevCmd.bat' -$compile = @( - 'set "PATH={0};{1};{2};%PATH%"' -f $nasmPath,$makePath,(Join-Path $gitRoot 'usr\bin') - 'call "{0}" -arch=x64 -host_arch=x64 >nul' -f $developerCommand - 'cd /d "{0}"' -f $buildPath - '"{0}" "{1}" --target=x86_64-win64-vs17 --as=nasm --disable-vp9 --disable-vp8-decoder --disable-examples --disable-unit-tests --disable-tools --disable-docs --disable-webm-io --disable-libyuv --disable-postproc --enable-realtime-only --enable-static-msvcrt' -f $bashPath,((Join-Path $sourcePath 'configure').Replace('\','/')) - 'make.exe -B vpx.vcxproj' - 'msbuild vpx.vcxproj /nologo /t:Rebuild /p:Configuration=Release /p:Platform=x64 /m:4 /v:minimal' -) -join ' && ' -& cmd.exe /d /s /c $compile | Out-Host -if ($LASTEXITCODE -ne 0 -or !(Test-Path $libraryPath)) { - throw 'Native VP8 library compilation failed.' -} -[IO.File]::WriteAllText($stampPath, $stamp + "`n") -return @{ Include = $sourcePath; Library = $libraryPath } diff --git a/native/capture/windows/build.ps1 b/native/capture/windows/build.ps1 index 4de4c23..ff80969 100644 --- a/native/capture/windows/build.ps1 +++ b/native/capture/windows/build.ps1 @@ -9,12 +9,14 @@ Set-StrictMode -Version Latest $ErrorActionPreference = 'Stop' $helperDirectory = Split-Path -Parent $MyInvocation.MyCommand.Path -$repositoryRoot = [IO.Path]::GetFullPath((Join-Path $helperDirectory '..\..')) +$repositoryRoot = [IO.Path]::GetFullPath((Join-Path $helperDirectory '..\..\..')) $outputPath = [IO.Path]::GetFullPath($OutputDirectory) $repositoryPrefix = $repositoryRoot.TrimEnd([IO.Path]::DirectorySeparatorChar) + [IO.Path]::DirectorySeparatorChar +$buildPrefix = (Join-Path $repositoryRoot 'build') + [IO.Path]::DirectorySeparatorChar if ($outputPath.Equals($repositoryRoot, [StringComparison]::OrdinalIgnoreCase) -or - $outputPath.StartsWith($repositoryPrefix, [StringComparison]::OrdinalIgnoreCase)) { - throw 'OutputDirectory must be outside the repository.' + ($outputPath.StartsWith($repositoryPrefix, [StringComparison]::OrdinalIgnoreCase) -and + !$outputPath.StartsWith($buildPrefix, [StringComparison]::OrdinalIgnoreCase))) { + throw 'OutputDirectory must be below build/ or outside the repository.' } $vswhere = Join-Path ${env:ProgramFiles(x86)} 'Microsoft Visual Studio\Installer\vswhere.exe' @@ -24,39 +26,45 @@ if ([string]::IsNullOrWhiteSpace($installationPath)) { } $developerCommand = Join-Path $installationPath 'Common7\Tools\VsDevCmd.bat' -$mainSourcePath = Join-Path $helperDirectory 'main.cpp' -$audioSourcePath = Join-Path $helperDirectory 'process_audio.cpp' -$targetSourcePath = Join-Path $helperDirectory 'capture_target.cpp' -$vp8SourcePath = Join-Path $helperDirectory 'vp8_encoder.cpp' $executablePath = Join-Path $outputPath 'screener-client-capture.exe' -$mainObjectPath = Join-Path $outputPath 'window-capture.obj' -$audioObjectPath = Join-Path $outputPath 'process-audio.obj' -$targetObjectPath = Join-Path $outputPath 'capture-target.obj' -$vp8ObjectPath = Join-Path $outputPath 'vp8-encoder.obj' New-Item -ItemType Directory -Path $outputPath -Force | Out-Null -$vpx = & (Join-Path $helperDirectory 'build-libvpx.ps1') -OutputDirectory (Join-Path $outputPath 'libvpx') -VisualStudioDirectory $installationPath - -$compile = @( - 'call "{0}" -arch=x64 -host_arch=x64 >nul' -f $developerCommand - 'cl.exe /nologo /c /std:c++20 /EHsc /W4 /WX /external:I "{2}" /external:W0 /DUNICODE /D_UNICODE /DWIN32_LEAN_AND_MEAN /D_WIN32_WINNT=0x0A00 /DNTDDI_VERSION=0x0A00000A "{0}" /Fo:"{1}"' -f $mainSourcePath, $mainObjectPath, $vpx.Include - 'cl.exe /nologo /c /std:c++20 /EHsc /W4 /WX /DUNICODE /D_UNICODE /DWIN32_LEAN_AND_MEAN /D_WIN32_WINNT=0x0A00 /DNTDDI_VERSION=0x0A00000A "{0}" /Fo:"{1}"' -f $audioSourcePath, $audioObjectPath - 'cl.exe /nologo /c /std:c++20 /EHsc /W4 /WX /DUNICODE /D_UNICODE /DWIN32_LEAN_AND_MEAN /D_WIN32_WINNT=0x0A00 /DNTDDI_VERSION=0x0A00000A "{0}" /Fo:"{1}"' -f $targetSourcePath, $targetObjectPath - 'cl.exe /nologo /c /std:c++20 /EHsc /O2 /W4 /WX /external:I "{2}" /external:W0 "{0}" /Fo:"{1}"' -f $vp8SourcePath, $vp8ObjectPath, $vpx.Include - 'link.exe /nologo /LTCG /out:"{0}" "{1}" "{2}" "{3}" "{4}" "{5}" ole32.lib mmdevapi.lib runtimeobject.lib user32.lib gdi32.lib dwmapi.lib shell32.lib mfplat.lib mf.lib mfuuid.lib d3d11.lib dxgi.lib dxguid.lib evr.lib oleaut32.lib windowsapp.lib' -f $executablePath, $mainObjectPath, $audioObjectPath, $targetObjectPath, $vp8ObjectPath, $vpx.Library -) -join ' && ' - -& cmd.exe /d /s /c $compile -if ($LASTEXITCODE -ne 0) { - throw "Window-capture helper compilation failed with exit code $LASTEXITCODE." +$webrtc = & (Join-Path $helperDirectory 'get-webrtc.ps1') +$includeFlags = @('', 'third_party\abseil-cpp', 'third_party\boringssl\src\include', + 'third_party\libyuv\include', 'third_party\libvpx\source\libvpx') | ForEach-Object { + '/external:I "{0}"' -f (Join-Path $webrtc.Include $_).TrimEnd('\') } +$compileFlags = '/nologo /c /std:c++20 /EHsc /GR /O2 /W4 /WX /MT /D_ITERATOR_DEBUG_LEVEL=0 /DUNICODE /D_UNICODE /DWIN32_LEAN_AND_MEAN /D_WIN32_WINNT=0x0A00 /DNTDDI_VERSION=0x0A00000A /external:W0 ' + ($includeFlags -join ' ') +$systemLibraries = 'ole32.lib mmdevapi.lib runtimeobject.lib user32.lib gdi32.lib dwmapi.lib shell32.lib mfplat.lib mf.lib mfuuid.lib d3d11.lib dxgi.lib dxguid.lib evr.lib oleaut32.lib windowsapp.lib winmm.lib ws2_32.lib strmiids.lib crypt32.lib dmoguids.lib iphlpapi.lib msdmo.lib secur32.lib wmcodecdspuuid.lib' +$linkCommand = '"{0}" /nologo /libpath:"{1}"' -f $webrtc.Linker,$webrtc.RuntimeLibraries + +function Invoke-CaptureBuild([string]$command) { + & cmd.exe /d /s /c ('call "{0}" -arch=x64 -host_arch=x64 >nul && {1}' -f $developerCommand,$command) | Out-Host + if ($LASTEXITCODE -ne 0) { + throw "Window-capture helper compilation failed with exit code $LASTEXITCODE." + } +} + +$objects = foreach ($name in @('main', 'process_audio', 'capture_target', 'h264_encoder', 'adaptive_encoder')) { + $source = Join-Path $helperDirectory ($name + '.cpp') + $object = Join-Path $outputPath ($name + '.obj') + $definitions = if ($name -eq 'adaptive_encoder') { '/DWEBRTC_WIN /DRTC_ENABLE_H265 /DNOMINMAX' } else { '' } + Invoke-CaptureBuild ('cl.exe {0} /DNDEBUG {1} "{2}" /Fo:"{3}"' -f $compileFlags,$definitions,$source,$object) + '"{0}"' -f $object +} +Invoke-CaptureBuild ('{0} /out:"{1}" {2} "{3}" {4}' -f $linkCommand,$executablePath,($objects -join ' '),$webrtc.Library,$systemLibraries) + if ($Check) { - $geometrySource = Join-Path $helperDirectory 'capture_geometry.test.cpp' - $geometryObject = Join-Path $outputPath 'capture-geometry.test.obj' - $geometryCheck = Join-Path $outputPath 'capture-geometry.test.exe' - $compileCheck = 'call "{0}" -arch=x64 -host_arch=x64 >nul && cl.exe /nologo /std:c++20 /EHsc /W4 /WX "{1}" /Fo:"{2}" /Fe:"{3}" /link user32.lib' -f $developerCommand, $geometrySource, $geometryObject, $geometryCheck - & cmd.exe /d /s /c $compileCheck - if ($LASTEXITCODE -ne 0) { throw 'Capture geometry check compilation failed.' } - & $geometryCheck - if ($LASTEXITCODE -ne 0) { throw 'Capture geometry check failed.' } + $vp8Object = Join-Path $outputPath 'vp8_encoder.obj' + Invoke-CaptureBuild ('cl.exe {0} /DNDEBUG "{1}" /Fo:"{2}"' -f $compileFlags,(Join-Path $helperDirectory 'vp8_encoder.cpp'),$vp8Object) + foreach ($name in @('capture_geometry', 'capture_control')) { + $source = Join-Path $helperDirectory ($name + '.test.cpp') + $object = Join-Path $outputPath ($name + '.test.obj') + $executable = Join-Path $outputPath ($name + '.test.exe') + Invoke-CaptureBuild ('cl.exe {0} "{1}" /Fo:"{2}"' -f $compileFlags,$source,$object) + $codecObjects = if ($name -eq 'capture_control') { '"{0}" "{1}"' -f $vp8Object,$webrtc.Library } else { '' } + Invoke-CaptureBuild ('{0} /out:"{1}" "{2}" {3} {4}' -f $linkCommand,$executable,$object,$codecObjects,$systemLibraries) + & $executable + if ($LASTEXITCODE -ne 0) { throw "Capture check failed: $name" } + } } Write-Output $executablePath diff --git a/native/capture/windows/capture_control.h b/native/capture/windows/capture_control.h new file mode 100644 index 0000000..a56a0e1 --- /dev/null +++ b/native/capture/windows/capture_control.h @@ -0,0 +1,58 @@ +#pragma once + +#include +#include +#include +#include +#include + +namespace screener::capture { + +constexpr int kMaxOutputs = 6; + +struct CaptureControl final { + char kind = 'Q'; + int layer = -1; + uint32_t value = 0; +}; + +// Control payload grammar inside one SMED input envelope. +class CaptureControls final { + public: + static CaptureControl Parse(std::string_view text) { + if (text.empty() || text.size() > 64) throw std::runtime_error("Invalid capture control size"); + for (unsigned char byte : text) { + if (byte < 32 || byte > 126) throw std::runtime_error("Invalid capture control text"); + } + std::istringstream input{std::string(text)}; + CaptureControl control; + std::string kind; + int64_t value = 0; + bool valid = static_cast(input >> kind) && kind.size() == 1; + control.kind = kind.empty() ? 0 : kind.front(); + switch (control.kind) { + case 'Q': break; + case 'K': + valid = valid && static_cast(input >> control.layer) && + control.layer >= -1 && control.layer < kMaxOutputs; + break; + case 'A': + valid = valid && static_cast(input >> control.layer >> value) && + control.layer >= 0 && control.layer < kMaxOutputs && value >= 0 && value <= 1; + control.value = static_cast(value); + break; + case 'B': + valid = valid && static_cast(input >> control.layer >> value) && + control.layer >= 0 && control.layer < kMaxOutputs && value >= 1'000 && + value <= UINT32_MAX; + control.value = static_cast(value); + break; + default: valid = false; + } + input >> std::ws; + if (!valid || !input.eof()) throw std::runtime_error("Invalid capture control command"); + return control; + } +}; + +} // namespace screener::capture diff --git a/native/capture/windows/capture_control.test.cpp b/native/capture/windows/capture_control.test.cpp new file mode 100644 index 0000000..5d917e0 --- /dev/null +++ b/native/capture/windows/capture_control.test.cpp @@ -0,0 +1,244 @@ +#include "capture_control.h" +#include "capture_input.h" +#include "capture_output.h" +#include "output_mailbox.h" +#include "vp8_encoder.h" +#include "vp8_decoder.h" + +#include +#include +#include +#include +#include +#include + +void CheckOutputPipe() { + HANDLE input = nullptr, output = nullptr; + assert(CreatePipe(&input, &output, nullptr, 4096)); + screener::capture::ProtocolWriter writer(output); + assert(SUCCEEDED(writer.WriteBegin(333'333, 333'333))); + const std::array payload{1, 2, 3}; + assert(SUCCEEDED(writer.Write(screener::capture::OutputKind::vp8, 1, + 333'333, 333'333, payload.data(), static_cast(payload.size()), 1, 320, 180))); + std::array data{}; + DWORD received = 0; + assert(ReadFile(input, data.data(), static_cast(data.size()), &received, nullptr)); + assert(received == data.size() && data[4] == 2 && data[5] == 5 && data[31] == 0); + assert(data[37] == 4 && data[38] == 1 && data[39] == 1); + assert(data[56] == 1 && data[57] == 64 && data[58] == 0 && data[59] == 180 && data[63] == 3); + + std::vector large(256 * 1024, 128); + HRESULT result = S_OK; + std::thread blocked([&]() { + result = writer.Write(screener::capture::OutputKind::vp8, 0, + 666'666, 333'333, large.data(), static_cast(large.size()), 1, 320, 180); + }); + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2); + DWORD available = 0; + while (available <= 32) { + assert(std::chrono::steady_clock::now() < deadline); + assert(PeekNamedPipe(input, nullptr, 0, nullptr, &available, nullptr)); + std::this_thread::yield(); + } + assert(WaitForSingleObject(blocked.native_handle(), 0) == WAIT_TIMEOUT); + writer.Stop(); + do { + CancelSynchronousIo(blocked.native_handle()); + assert(std::chrono::steady_clock::now() < deadline); + } while (WaitForSingleObject(blocked.native_handle(), 10) == WAIT_TIMEOUT); + blocked.join(); + assert(FAILED(result)); + CloseHandle(input); + CloseHandle(output); +} + +void CheckWorkers() { + struct Input { uint32_t index; }; + using Mailbox = screener::capture::OutputMailbox; + std::array mailboxes{Mailbox(90'000), Mailbox(300'000), Mailbox(1'200'000)}; + std::array workers; + std::array last{}, count{}, retired{}; + std::array recovery{}; + std::mutex mutex; + std::condition_variable changed; + std::exception_ptr failure; + std::promise entered, release; + auto entered_future = entered.get_future(); + auto released = release.get_future().share(); + for (size_t layer = 0; layer < workers.size(); ++layer) { + workers[layer] = std::thread([&, layer]() { + try { + const uint32_t width = 160u << layer; + const uint32_t height = 90u << layer; + std::vector pixels(static_cast(width) * height * 3 / 2, 128); + std::unique_ptr encoder; + for (;;) { + const auto work = mailboxes[layer].Take(); + if (work.action == Mailbox::Action::stop) break; + if (work.action == Mailbox::Action::retire) { + encoder.reset(); + std::lock_guard lock(mutex); + ++retired[layer]; + changed.notify_one(); + continue; + } + if (layer == 0 && work.input->index == 1) { + entered.set_value(); + released.wait(); + } + if (!encoder) encoder = std::make_unique(width, height, 30, work.bitrate); + encoder->SetBitrate(work.bitrate); + const auto output = encoder->Encode(pixels.data(), width, + static_cast(work.input->index) * 10'000'000 / 30, work.recovery); + if (!mailboxes[layer].Accept(work.generation)) continue; + std::lock_guard lock(mutex); + last[layer] = work.input->index; + recovery[layer] = output.key_frame; + ++count[layer]; + changed.notify_one(); + } + } catch (...) { + std::lock_guard lock(mutex); + failure = std::current_exception(); + changed.notify_one(); + } + }); + } + auto submit = [&](uint32_t index) { + auto input = std::make_shared(Input{index}); + for (auto& mailbox : mailboxes) mailbox.Submit(input); + }; + auto wait = [&](auto predicate) { + std::unique_lock lock(mutex); + const bool ready = changed.wait_for(lock, std::chrono::seconds(2), [&]() { return failure || predicate(); }); + assert(ready && !failure); + }; + submit(1); + assert(entered_future.wait_for(std::chrono::seconds(2)) == std::future_status::ready); + for (uint32_t index = 2; index <= 4; ++index) submit(index); + wait([&]() { return last[1] == 4 && last[2] == 4; }); + mailboxes[0].RequestKeyFrame(); + release.set_value(); + wait([&]() { return last[0] == 4; }); + assert(count[0] == 2 && recovery[0]); // New request survived the older keyframe; pending 2/3 were replaced. + mailboxes[1].SetActive(false); + mailboxes[2].SetActive(false); + submit(5); + wait([&]() { return last[0] == 5 && retired[1] == 1 && retired[2] == 1; }); + assert(last[1] == 4 && last[2] == 4); + mailboxes[0].SetBitrate(30'000); + for (auto& mailbox : mailboxes) { mailbox.SetActive(true); mailbox.RequestKeyFrame(); } + submit(6); + wait([&]() { return last[0] == 6 && last[1] == 6 && last[2] == 6; }); + assert(recovery[0] && recovery[1] && recovery[2]); + const auto stopped = std::chrono::steady_clock::now(); + for (auto& mailbox : mailboxes) mailbox.Stop(); + for (auto& worker : workers) worker.join(); + assert(std::chrono::steady_clock::now() - stopped < std::chrono::seconds(2)); +} + +void CheckDroppedRecoveryInput() { + using Mailbox = screener::capture::OutputMailbox; + Mailbox mailbox(90'000); + mailbox.Submit(std::make_shared(1)); + const auto dropped = mailbox.Take(); + assert(dropped.recovery && *dropped.input == 1); + // The codec accepted input but produced no AU; preserve its consumed request. + if (dropped.recovery) mailbox.RequestKeyFrame(); + mailbox.Submit(std::make_shared(2)); + const auto recovery = mailbox.Take(); + assert(recovery.recovery && *recovery.input == 2); + assert(mailbox.Accept(recovery.generation)); + mailbox.Submit(std::make_shared(3)); + assert(!mailbox.Take().recovery); + mailbox.Stop(); +} + +void CheckIndependentActivation() { + using Mailbox = screener::capture::OutputMailbox; + Mailbox original(300'000), extra(90'000, false); + original.Submit(std::make_shared(1)); + const auto original_frame = original.Take(); + extra.Submit(std::make_shared(1)); + auto pending = std::async(std::launch::async, [&]() { return extra.Take(); }); + assert(pending.wait_for(std::chrono::milliseconds(20)) == std::future_status::timeout); + assert(extra.SetActive(true)); + assert(!extra.SetActive(true)); + extra.SetBitrate(30'000); + extra.Submit(std::make_shared(2)); + assert(pending.wait_for(std::chrono::seconds(2)) == std::future_status::ready); + const auto activated = pending.get(); + assert(activated.action == Mailbox::Action::frame && *activated.input == 2 && + activated.recovery && activated.bitrate == 30'000); + assert(!extra.SetActive(false)); + assert(!extra.Accept(activated.generation) && original.Accept(original_frame.generation)); + assert(extra.Take().action == Mailbox::Action::retire); + assert(extra.SetActive(true)); + extra.Submit(std::make_shared(3)); + const auto replacement = extra.Take(); + assert(replacement.recovery && *replacement.input == 3 && + replacement.generation != activated.generation); + original.Stop(); + extra.Stop(); +} + +void CheckInputEnvelopes() { + std::string control(32, '\0'); + control.replace(0, 4, "SMED"); + control[4] = 2; + control[5] = 7; + control[31] = 4; + control += "K -1"; + screener::capture::InputEnvelopes parser; + assert(parser.Feed(std::string_view(control).substr(0, 11)).empty()); + assert(!parser.Empty()); + const auto controls = parser.Feed(std::string_view(control).substr(11)); + assert(controls.size() == 1 && controls[0].Control().kind == 'K' && controls[0].Control().layer == -1); + assert(parser.Empty()); + auto invalid = control; + invalid[7] = 1; + bool rejected = false; + try { (void)parser.Feed(invalid); } catch (const std::runtime_error&) { rejected = true; } + assert(rejected); +} + +int main() { + using screener::capture::CaptureControls; + assert(CaptureControls::Parse("K -1").layer == -1); + assert(CaptureControls::Parse("A 5 1").layer == 5); + assert(CaptureControls::Parse("A 5 1").value == 1); + assert(CaptureControls::Parse("A 0 0").value == 0); + assert(CaptureControls::Parse("K 5").layer == 5); + assert(CaptureControls::Parse("B 1 90000").value == 90'000); + assert(CaptureControls::Parse("Q").kind == 'Q'); + for (const auto& bad : {"K 6", "A 2", "A 6 1", "A -1 1", "A 0 2", "A2", "B 6 1000", "B 0 -1", "Q ignored", "\n"}) { + bool rejected = false; + try { (void)CaptureControls::Parse(bad); } + catch (const std::runtime_error&) { rejected = true; } + assert(rejected); + } + bool rejected = false; + try { (void)CaptureControls::Parse(std::string(65, 'K')); } + catch (const std::runtime_error&) { rejected = true; } + assert(rejected); + + screener::capture::Vp8Encoder encoder(160, 90, 30, 90'000); + screener::capture::Vp8Decoder decoder; + std::vector pixels(160 * 90 * 3 / 2, 128); + for (uint32_t frame = 0; frame < 12; ++frame) { + if (frame == 4) encoder.SetBitrate(30'000); + if (frame == 8) encoder.SetBitrate(90'000); + pixels[frame] = static_cast(frame); + const bool recovery = frame % 4 == 0; + const auto output = encoder.Encode(pixels.data(), 160, + (static_cast(frame) + 1) * 10'000'000 / 30, recovery); + assert(!output.bytes.empty() && (!recovery || output.key_frame)); + const auto decoded = decoder.Decode(output.bytes); + assert(decoded.width == 160 && decoded.height == 90 && decoded.nv12.size() == pixels.size()); + } + CheckWorkers(); + CheckDroppedRecoveryInput(); + CheckIndependentActivation(); + CheckOutputPipe(); + CheckInputEnvelopes(); +} diff --git a/native/capture/windows/capture_input.h b/native/capture/windows/capture_input.h new file mode 100644 index 0000000..1acb8b7 --- /dev/null +++ b/native/capture/windows/capture_input.h @@ -0,0 +1,101 @@ +#pragma once + +#include "capture_control.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace screener::capture { + +struct InputEnvelope final { + uint8_t kind = 0; + bool recovery = false; + uint64_t timestamp = 0; + uint64_t duration = 0; + uint16_t width = 0; + uint16_t height = 0; + std::vector data; + + CaptureControl Control() const { + if (kind != 7) throw std::runtime_error("Expected a capture control envelope"); + return CaptureControls::Parse(std::string_view( + reinterpret_cast(data.data()), data.size())); + } +}; + +// The private stdin contract has one framing format for controls and encoded AUs. +// Retain at most one incomplete, bounded envelope between pipe reads. +class InputEnvelopes final { + public: + bool Empty() const { return header_size_ == 0 && frame_.data.empty(); } + + std::vector Feed(std::string_view bytes) { + std::vector frames; + while (!bytes.empty()) { + if (header_size_ < header_.size()) { + const size_t take = std::min(header_.size() - header_size_, bytes.size()); + std::copy_n(bytes.data(), take, header_.data() + header_size_); + header_size_ += take; + bytes.remove_prefix(take); + if (header_size_ != header_.size()) break; + ReadHeader(); + } + const size_t take = std::min(payload_size_ - frame_.data.size(), bytes.size()); + frame_.data.insert(frame_.data.end(), bytes.data(), bytes.data() + take); + bytes.remove_prefix(take); + if (frame_.data.size() == payload_size_) { + if (frame_.kind == 7) (void)frame_.Control(); + frames.push_back(std::move(frame_)); + frame_ = {}; + header_size_ = 0; + } + } + return frames; + } + + private: + uint64_t ReadBE(size_t offset, size_t count) const { + uint64_t result = 0; + for (size_t index = 0; index < count; ++index) result = result << 8 | header_[offset + index]; + return result; + } + + void ReadHeader() { + frame_.kind = header_[5]; + frame_.recovery = header_[6] == 1; + frame_.timestamp = ReadBE(8, 8); + frame_.duration = ReadBE(16, 8); + frame_.width = static_cast(ReadBE(24, 2)); + frame_.height = static_cast(ReadBE(26, 2)); + payload_size_ = static_cast(ReadBE(28, 4)); + if (header_[0] != 'S' || header_[1] != 'M' || header_[2] != 'E' || header_[3] != 'D' || + header_[4] != 2 || header_[7] != 0 || payload_size_ == 0) { + throw std::runtime_error("Invalid native input envelope"); + } + if (frame_.kind == 7) { + if (header_[6] != 0 || frame_.timestamp != 0 || frame_.duration != 0 || + frame_.width != 0 || frame_.height != 0 || payload_size_ > 64) { + throw std::runtime_error("Invalid native control envelope"); + } + } else if ((frame_.kind != 2 && frame_.kind != 4) || header_[6] > 1 || + frame_.duration == 0 || frame_.timestamp > static_cast(INT64_MAX) / 100 || + frame_.duration > static_cast(INT64_MAX) / 100 || + frame_.width < 2 || frame_.height < 2 || frame_.width > 2560 || frame_.height > 1440 || + (frame_.width & 1) || (frame_.height & 1) || payload_size_ > 4 * 1024 * 1024) { + throw std::runtime_error("Invalid native encoded input"); + } + frame_.data.reserve(payload_size_); + } + + std::array header_{}; + size_t header_size_ = 0; + size_t payload_size_ = 0; + InputEnvelope frame_; +}; + +} // namespace screener::capture diff --git a/native/capture/windows/capture_output.h b/native/capture/windows/capture_output.h new file mode 100644 index 0000000..50bd0f1 --- /dev/null +++ b/native/capture/windows/capture_output.h @@ -0,0 +1,88 @@ +#pragma once + +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include + +#include +#include +#include +#include +#include + +namespace screener::capture { + +constexpr DWORD kMaxProductAccessUnitBytes = 4 * 1024 * 1024; +constexpr DWORD kMaxStatusBytes = 4 * 1024; +enum class OutputKind : UINT8 { pcm = 1, h264 = 2, status = 3, vp8 = 4, begin = 5, unavailable = 6 }; + +// The caller owns the handle. One lock keeps each SMED envelope contiguous. +class ProtocolWriter final { + public: + explicit ProtocolWriter(HANDLE output = GetStdHandle(STD_OUTPUT_HANDLE)) : output_(output) {} + void Stop() { stopped_.store(true); } + + HRESULT Write(OutputKind kind, UINT8 flags, UINT64 timestamp100ns, + UINT64 duration100ns, const BYTE* data, DWORD size, + UINT8 layer = 0, UINT16 width = 0, UINT16 height = 0) { + const bool diagnostic = kind == OutputKind::status || kind == OutputKind::unavailable; + const DWORD maximum = diagnostic ? kMaxStatusBytes : kMaxProductAccessUnitBytes; + if (size > maximum || (kind == OutputKind::begin ? size != 0 : data == nullptr || size == 0) || + (!diagnostic && duration100ns == 0)) return HRESULT_FROM_WIN32(ERROR_INVALID_DATA); + std::array header{'S', 'M', 'E', 'D', 2, static_cast(kind), flags, layer}; + PutBE(header.data() + 8, timestamp100ns, 8); + PutBE(header.data() + 16, duration100ns, 8); + PutBE(header.data() + 24, width, 2); + PutBE(header.data() + 26, height, 2); + PutBE(header.data() + 28, size, 4); + std::lock_guard lock(mutex_); + const HRESULT result = WriteAll(header.data(), static_cast(header.size())); + return SUCCEEDED(result) ? WriteAll(data, size) : result; + } + + HRESULT WriteStatus(const std::string& value) { + return Write(OutputKind::status, 0, 0, 0, reinterpret_cast(value.data()), + static_cast(value.size())); + } + + HRESULT WriteBegin(UINT64 timestamp100ns, UINT64 duration100ns) { + return Write(OutputKind::begin, 0, timestamp100ns, duration100ns, nullptr, 0); + } + + HRESULT WriteUnavailable(UINT8 layer, std::string detail) { + detail.resize(std::min(detail.size(), kMaxStatusBytes)); + for (char& value : detail) { + if (static_cast(value) < 32 || static_cast(value) > 126) value = '?'; + } + if (detail.empty()) detail = "Output encoder is unavailable"; + return Write(OutputKind::unavailable, 0, 0, 0, reinterpret_cast(detail.data()), + static_cast(detail.size()), layer); + } + + private: + static void PutBE(BYTE* output, UINT64 value, int size) { + for (int index = size - 1; index >= 0; --index) { + output[index] = static_cast(value & 0xff); + value >>= 8; + } + } + + HRESULT WriteAll(const BYTE* data, DWORD size) { + while (size > 0) { + if (stopped_.load()) return HRESULT_FROM_WIN32(ERROR_OPERATION_ABORTED); + DWORD written = 0; + if (!WriteFile(output_, data, size, &written, nullptr)) return HRESULT_FROM_WIN32(GetLastError()); + if (written == 0) return HRESULT_FROM_WIN32(ERROR_WRITE_FAULT); + data += written; + size -= written; + } + return S_OK; + } + + const HANDLE output_; + std::mutex mutex_; + std::atomic stopped_{false}; +}; + +} // namespace screener::capture diff --git a/native/capture/windows/encoded_group.fixture.cpp b/native/capture/windows/encoded_group.fixture.cpp new file mode 100644 index 0000000..880d66e --- /dev/null +++ b/native/capture/windows/encoded_group.fixture.cpp @@ -0,0 +1,87 @@ +#include "vp8_encoder.h" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace { + +using Clock = std::chrono::steady_clock; +using screener::capture::Vp8Encoder; +struct Profile { uint32_t width, height, bitrate; }; +// Codec fixtures, not product presets. Layers are L=0, M=1, H=2. +constexpr std::array kProfiles{{ + {160, 90, 90'000}, {320, 180, 300'000}, {640, 360, 1'200'000}}}; +// Eight source frames per pair: H/H, H/L, L/L, M/M, H/H. +constexpr std::array, 5> kTargets{{ + {2, 2}, {2, 0}, {0, 0}, {1, 1}, {2, 2}}}; + +void DrawFrame(std::vector& pixels, Profile profile, uint32_t frame) { + pixels.assign(static_cast(profile.width) * profile.height * 3 / 2, 128); + for (uint32_t y = 0; y < profile.height; ++y) { + for (uint32_t x = 0; x < profile.width; ++x) { + const uint32_t scene_x = (x * 640 / profile.width + frame * 8) % 640; + const uint32_t scene_y = y * 360 / profile.height; + pixels[static_cast(y) * profile.width + x] = + static_cast(32 + (((scene_x / 16) ^ (scene_y / 16)) % 8) * 24); + } + } +} + +void WriteFrame(uint32_t index, int layer, Profile profile, uint64_t pts, + const screener::capture::Vp8Frame& packet) { + std::cout << "{\"FrameIndex\":" << index << ",\"Layer\":" << layer + << ",\"Width\":" << profile.width << ",\"Height\":" << profile.height + << ",\"PTS\":" << pts << ",\"Recovery\":" + << (packet.key_frame ? "true" : "false") << ",\"DataHex\":\""; + constexpr char hex[] = "0123456789abcdef"; + for (uint8_t byte : packet.bytes) { + std::cout.put(hex[byte >> 4]); + std::cout.put(hex[byte & 15]); + } + std::cout << "\"}\n"; + if (!std::cout) throw std::runtime_error("Fixture output failed"); +} + +} // namespace + +int main() { + try { + const auto deadline = Clock::now() + std::chrono::seconds(15); + std::array, 3> encoders; + std::vector pixels; + for (uint32_t frame = 0; frame < 40; ++frame) { + const auto& targets = kTargets[frame / 8]; + const int highest = std::max(targets[0], targets[1]); + const uint64_t timestamp100ns = (static_cast(frame) + 1) * 10'000'000 / 30; + // Serial CPU fixture only; this does not implement production worker isolation. + for (int layer = 2; layer >= 0; --layer) { + if (Clock::now() >= deadline) throw std::runtime_error("Fixture deadline exceeded"); + if (layer > highest) { + encoders[layer].reset(); + continue; + } + const auto profile = kProfiles[layer]; + const bool recovery = frame % 8 == 0 || !encoders[layer]; + if (!encoders[layer]) { + encoders[layer] = std::make_unique( + profile.width, profile.height, 30, profile.bitrate); + } + DrawFrame(pixels, profile, frame); + const auto packet = encoders[layer]->Encode( + pixels.data(), profile.width, timestamp100ns, recovery); + if (Clock::now() >= deadline) throw std::runtime_error("Fixture deadline exceeded"); + WriteFrame(frame, layer, profile, timestamp100ns * 100, packet); + } + } + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/native/capture/windows/encoded_variants.probe.cpp b/native/capture/windows/encoded_variants.probe.cpp new file mode 100644 index 0000000..675ccde --- /dev/null +++ b/native/capture/windows/encoded_variants.probe.cpp @@ -0,0 +1,276 @@ +#include "vp8_encoder.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef _WIN32 +#define NOMINMAX +#include +#include +#endif + +using Clock = std::chrono::steady_clock; +using screener::capture::Vp8Encoder; + +namespace { + +constexpr int kFps = 30; +constexpr int kClipFrames = 32; +constexpr double kFrameMs = 1000.0 / kFps; +// Measurement fixtures, not an application bitrate ladder. +struct Profile { unsigned width, height, bitrate; }; +constexpr std::array kProfiles{{ + {480, 270, 312'500}, {960, 540, 1'250'000}, {1920, 1080, 5'000'000}}}; +using Clip = std::vector>; + +double Milliseconds(Clock::duration duration) { + return std::chrono::duration(duration).count(); +} + +double CpuMilliseconds() { +#ifdef _WIN32 + FILETIME created{}, exited{}, kernel{}, user{}; + if (!GetProcessTimes(GetCurrentProcess(), &created, &exited, &kernel, &user)) { + throw std::runtime_error("GetProcessTimes failed"); + } + auto ticks = [](FILETIME value) { + return (static_cast(value.dwHighDateTime) << 32) | value.dwLowDateTime; + }; + return static_cast(ticks(kernel) + ticks(user)) / 10'000; +#else + return 1000.0 * std::clock() / CLOCKS_PER_SEC; +#endif +} + +uint64_t CpuCycles() { +#ifdef _WIN32 + ULONG64 cycles = 0; + if (!QueryProcessCycleTime(GetCurrentProcess(), &cycles)) { + throw std::runtime_error("QueryProcessCycleTime failed"); + } + return cycles; +#else + return 0; +#endif +} + +size_t ResidentBytes() { +#ifdef _WIN32 + PROCESS_MEMORY_COUNTERS counters{}; + if (!GetProcessMemoryInfo(GetCurrentProcess(), &counters, sizeof(counters))) { + throw std::runtime_error("GetProcessMemoryInfo failed"); + } + return counters.WorkingSetSize; +#else + return 0; +#endif +} + +Clip MakeClip(Profile profile, bool motion) { + Clip clip(kClipFrames); + for (int frame = 0; frame < kClipFrames; ++frame) { + auto& pixels = clip[frame]; + pixels.assign(static_cast(profile.width) * profile.height * 3 / 2, 128); + for (unsigned y = 0; y < profile.height; ++y) { + for (unsigned x = 0; x < profile.width; ++x) { + const unsigned shift = motion || (x < profile.width / 16 && y < profile.height / 16) + ? static_cast(frame) * 4 : 0; + const unsigned u = (x * 1920 / profile.width + shift) % 128; + const unsigned v = (y * 1080 / profile.height) % 128; + pixels[static_cast(y) * profile.width + x] = + static_cast(32 + ((u / 8) ^ (v / 8)) * 8 + ((u ^ v) & 31) * 2); + } + } + } + return clip; +} + +struct Stats { + std::vector work; + double cpu_ms = 0; + uint64_t cpu_cycles = 0; + double init_ms = 0; + int creates = 0; + std::array encodes{}; + std::array bytes{}; +}; + +template +void PrintArray(const std::array& values) { + std::cout << '['; + for (size_t i = 0; i < N; ++i) std::cout << (i ? "," : "") << values[i]; + std::cout << ']'; +} + +void PrintStats(const Stats& stats) { + auto sorted = stats.work; + std::sort(sorted.begin(), sorted.end()); + const double media_ms = static_cast(sorted.size()) * kFrameMs; + std::cout << "{\"sourceFrames\":" << sorted.size() + << ",\"workMeanMs\":" << std::accumulate(sorted.begin(), sorted.end(), 0.0) / sorted.size() + << ",\"workP95Ms\":" << sorted[static_cast(std::ceil(sorted.size() * 0.95)) - 1] + << ",\"workOverFrameBudget\":" << std::count_if(sorted.begin(), sorted.end(), [](double value) { return value > kFrameMs; }) + << ",\"cpuMs\":" << stats.cpu_ms + << ",\"cpuCyclesPerFrame\":" << static_cast(stats.cpu_cycles) / sorted.size() + << ",\"cpuCoresAt30Fps\":" << stats.cpu_ms / media_ms + << ",\"createCount\":" << stats.creates << ",\"createMs\":" << stats.init_ms + << ",\"encodedFrames\":"; + PrintArray(stats.encodes); + std::cout << ",\"encodedBytes\":"; + PrintArray(stats.bytes); + std::cout << '}'; +} + +struct Transition { int frame, layer; double ready_ms; size_t bytes; }; + +void Run(const std::string& scenario, const std::string& scene) { + const bool warm = scenario == "exact-warm" || scenario == "warm-clock-ablation"; + const bool trace = scenario == "envelope" || scenario == "exact-cold" || warm; + unsigned fixed_mask = 0; + if (scenario == "high") fixed_mask = 4; + if (scenario == "two-high") fixed_mask = 12; + if (scenario == "high-low") fixed_mask = 5; + if (scenario == "all") fixed_mask = 7; + if (scenario == "mid-low") fixed_mask = 3; + if (scenario == "low") fixed_mask = 1; + if ((!trace && fixed_mask == 0) || (scene != "motion" && scene != "still")) { + throw std::runtime_error("Unknown scenario or scene"); + } + // Same scene at each size; preparation is excluded, no scaler is being measured. + std::array clips; + for (size_t i = 0; i < clips.size(); ++i) clips[i] = MakeClip(kProfiles[i], scene == "motion"); + const size_t input_resident = ResidentBytes(); + std::array, 4> encoders; + std::array codec_frames{}; + constexpr std::array, 7> demands{{ + {2, 2}, {2, 0}, {0, 0}, {1, 1}, {2, 2}, {2, 0}, {2, 2}}}; + const int frames = trace ? static_cast(demands.size()) * 60 : 120; + std::array previous_demand{-1, -1}; + unsigned previous_active = 0; + std::vector transitions; + std::vector phases(trace ? demands.size() : 1); + Stats total; + const auto started = Clock::now(); + double phase_cpu_start = CpuMilliseconds(); + uint64_t phase_cycles_start = CpuCycles(); + for (int frame = 0; frame < frames; ++frame) { + if (Clock::now() - started > std::chrono::seconds(30)) throw std::runtime_error("Probe deadline exceeded"); + const size_t phase = trace ? static_cast(frame / 60) : 0; + unsigned active = fixed_mask; + unsigned switched = 0; + if (trace) { + const auto& demand = demands[phase]; + active = (1u << demand[0]) | (1u << demand[1]); + if (scenario == "envelope") active = (1u << (std::max(demand[0], demand[1]) + 1)) - 1; + for (size_t child = 0; child < demand.size(); ++child) { + if (demand[child] != previous_demand[child]) switched |= 1u << demand[child]; + } + previous_demand = demand; + } + const bool measured = trace || frame >= 30; + const auto tick = Clock::now(); + Stats step; + // Serial scheduling keeps the probe bounded; parallel encoder scheduling is not measured. + for (int slot = 3; slot >= 0; --slot) { + const unsigned bit = 1u << slot; + if (!(active & bit)) { + if (!warm) encoders[slot].reset(); + continue; + } + const size_t layer = static_cast(std::min(slot, 2)); + const auto profile = kProfiles[layer]; + if (!encoders[slot]) { + const auto init = Clock::now(); + encoders[slot] = std::make_unique(profile.width, profile.height, kFps, profile.bitrate); + step.init_ms += Milliseconds(Clock::now() - init); + ++step.creates; + } + const bool keyframe = !(previous_active & bit) || (switched & bit); + // Isolate codec timestamp-gap effects only; not a proposed media clock policy. + const uint64_t pts_frame = scenario == "warm-clock-ablation" ? codec_frames[slot] : static_cast(frame); + auto packet = encoders[slot]->Encode(clips[layer][frame % kClipFrames].data(), profile.width, + pts_frame * 10'000'000 / kFps, keyframe); + ++codec_frames[slot]; + ++step.encodes[slot]; + step.bytes[slot] += packet.bytes.size(); + if (switched & bit) { + if (!packet.key_frame) throw std::runtime_error("Switch lacked a keyframe"); + transitions.push_back({frame, slot, Milliseconds(Clock::now() - tick), packet.bytes.size()}); + } + } + const double work_ms = Milliseconds(Clock::now() - tick); + if (measured) { + for (Stats* stats : {&total, &phases[phase]}) { + stats->work.push_back(work_ms); + stats->init_ms += step.init_ms; + stats->creates += step.creates; + for (size_t i = 0; i < step.encodes.size(); ++i) { + stats->encodes[i] += step.encodes[i]; + stats->bytes[i] += step.bytes[i]; + } + } + } + previous_active = active; + std::this_thread::sleep_until(started + std::chrono::microseconds(static_cast(frame + 1) * 1'000'000 / kFps)); + // Whole windows include worker spin/wakeup cost and avoid per-call CPU tick quantization. + if (!trace && frame == 29) { + phase_cpu_start = CpuMilliseconds(); + phase_cycles_start = CpuCycles(); + } + if ((trace && (frame + 1) % 60 == 0) || frame + 1 == frames) { + const double cpu_now = CpuMilliseconds(); + const uint64_t cycles_now = CpuCycles(); + phases[phase].cpu_ms = cpu_now - phase_cpu_start; + phases[phase].cpu_cycles = cycles_now - phase_cycles_start; + total.cpu_ms += phases[phase].cpu_ms; + total.cpu_cycles += phases[phase].cpu_cycles; + phase_cpu_start = cpu_now; + phase_cycles_start = cycles_now; + } + } + const size_t active_resident = ResidentBytes(); + for (auto& encoder : encoders) encoder.reset(); + const size_t stopped_resident = ResidentBytes(); + std::cout << std::fixed << std::setprecision(3) + << "{\"scenario\":\"" << scenario << "\",\"scene\":\"" << scene + << "\",\"codec\":\"" << Vp8Encoder::Version() << "\",\"summary\":"; + PrintStats(total); + std::cout << ",\"inputResidentBytes\":" << input_resident << ",\"activeResidentBytes\":" << active_resident + << ",\"stoppedResidentBytes\":" << stopped_resident << ",\"phases\":["; + for (size_t i = 0; i < phases.size(); ++i) { + if (i) std::cout << ','; + PrintStats(phases[i]); + } + std::cout << "],\"transitions\":["; + for (size_t i = 0; i < transitions.size(); ++i) { + const auto& value = transitions[i]; + if (i) std::cout << ','; + std::cout << "{\"frame\":" << value.frame << ",\"layer\":" << value.layer + << ",\"keyframeReadyMs\":" << value.ready_ms << ",\"keyframeBytes\":" << value.bytes << '}'; + } + std::cout << "]}\n"; +} + +} // namespace + +int main(int argc, char** argv) { + try { + if (argc != 3) throw std::runtime_error("Usage: encoded-variants.probe "); + Run(argv[1], argv[2]); + return 0; + } catch (const std::exception& error) { + std::cerr << error.what() << '\n'; + return 1; + } +} diff --git a/native/capture/windows/get-webrtc.ps1 b/native/capture/windows/get-webrtc.ps1 new file mode 100644 index 0000000..b8958a4 --- /dev/null +++ b/native/capture/windows/get-webrtc.ps1 @@ -0,0 +1,63 @@ +[CmdletBinding()] +param() + +Set-StrictMode -Version Latest +$ErrorActionPreference = 'Stop' +$ProgressPreference = 'SilentlyContinue' +$repositoryRoot = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..\..\..')) +$pins = Get-Content -Raw (Join-Path $PSScriptRoot 'webrtc-dependencies.json') | ConvertFrom-Json +$cachePath = Join-Path $repositoryRoot 'build\encoder-pool' +$sdkPath = Join-Path $cachePath 'webrtc' +$toolchainPath = Join-Path $cachePath 'msvc-crt' +New-Item -ItemType Directory -Path $cachePath,$toolchainPath -Force | Out-Null + +function Expand-PinnedArchive($pin, [string]$name, [string]$destination, [string]$entry) { + $archive = Join-Path $cachePath $name + if (!(Test-Path $archive) -or + (Get-FileHash $archive -Algorithm SHA256).Hash.ToLowerInvariant() -ne $pin.sha256) { + $download = $archive + '.download' + Invoke-WebRequest -UseBasicParsing -Uri $pin.url -OutFile $download -TimeoutSec 600 + if ((Get-FileHash $download -Algorithm SHA256).Hash.ToLowerInvariant() -ne $pin.sha256) { + throw "Native WebRTC dependency digest mismatch: $name" + } + Move-Item -LiteralPath $download -Destination $archive -Force + } + $stamp = $archive + '.extracted' + if (!(Test-Path (Join-Path $destination $entry)) -or !(Test-Path $stamp) -or + (Get-Content -Raw $stamp).Trim() -ne $pin.sha256) { + & tar -xf $archive -C $destination $entry + if ($LASTEXITCODE -ne 0) { throw "Could not unpack native WebRTC dependency: $name" } + [IO.File]::WriteAllText($stamp, $pin.sha256 + "`n") + } +} + +Expand-PinnedArchive $pins.webrtc 'webrtc.windows_x86_64.zip' $cachePath 'webrtc' +Expand-PinnedArchive $pins.msvc.crt 'msvc-crt.vsix' $toolchainPath ($pins.msvc.directory + '/lib/x64') +Expand-PinnedArchive $pins.msvc.tools 'msvc-tools.vsix' $toolchainPath ($pins.msvc.directory + '/bin/Hostx64/x64') + +$versions = Get-Content -Raw (Join-Path $sdkPath 'VERSIONS') | ConvertFrom-StringData +if ($versions.WEBRTC_COMMIT -ne $pins.webrtc.commit -or + ('m' + $versions.WEBRTC_BUILD_VERSION) -ne $pins.webrtc.version) { + throw 'Cached WebRTC SDK version differs from the dependency pin.' +} +$notices = Get-Content -Raw (Join-Path $repositoryRoot 'licenses\upstream.json') | ConvertFrom-Json +$notice = $notices.('webrtc@' + $pins.webrtc.version) +$packagedNotice = Join-Path $repositoryRoot ('licenses\' + $notice.file) +# Tracked notices use LF without trailing whitespace; the Windows SDK uses CRLF. +$sdkNotice = [IO.File]::ReadAllText((Join-Path $sdkPath 'NOTICE')).Replace("`r`n", "`n").TrimEnd() + "`n" +$sdkNotice = [regex]::Replace($sdkNotice, '(?m)[ \t]+$', '') +$sha256 = [Security.Cryptography.SHA256]::Create() +try { + $sdkNoticeHash = [BitConverter]::ToString($sha256.ComputeHash([Text.Encoding]::UTF8.GetBytes($sdkNotice))).Replace('-', '').ToLowerInvariant() +} finally { $sha256.Dispose() } +if ($sdkNoticeHash -ne $notice.sha256 -or + (Get-FileHash $packagedNotice -Algorithm SHA256).Hash.ToLowerInvariant() -ne $notice.sha256) { + throw 'WebRTC SDK and packaged notices must match the pinned upstream notice.' +} +$msvcPath = Join-Path $toolchainPath $pins.msvc.directory +return @{ + Include = Join-Path $sdkPath 'include' + Library = Join-Path $sdkPath 'lib\webrtc.lib' + RuntimeLibraries = Join-Path $msvcPath 'lib\x64' + Linker = Join-Path $msvcPath 'bin\Hostx64\x64\link.exe' +} diff --git a/native/capture/windows/h264_decoder.h b/native/capture/windows/h264_decoder.h new file mode 100644 index 0000000..d06e07b --- /dev/null +++ b/native/capture/windows/h264_decoder.h @@ -0,0 +1,171 @@ +#pragma once + +#include "capture_input.h" + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + +namespace screener::capture { + +// One inbox decoder owns the compressed reference chain. Consumers must finish +// copying/retaining an output sample before the callback returns. +class H264Decoder final { + public: + using Consumer = std::function; + + explicit H264Decoder(IMFDXGIDeviceManager* manager) { + Check(CoCreateInstance(__uuidof(CMSH264DecoderMFT), nullptr, CLSCTX_INPROC_SERVER, + IID_PPV_ARGS(&decoder_)), "create"); + Microsoft::WRL::ComPtr attributes; + Check(decoder_->GetAttributes(&attributes), "attributes"); + UINT32 aware = FALSE; + if (manager && SUCCEEDED(attributes->GetUINT32(MF_SA_D3D11_AWARE, &aware)) && aware) { + Check(decoder_->ProcessMessage(MFT_MESSAGE_SET_D3D_MANAGER, + reinterpret_cast(manager)), "device-manager"); + } + Microsoft::WRL::ComPtr codec; + Check(decoder_.As(&codec), "codec-api"); + for (const auto& setting : { + std::pair{CODECAPI_AVLowLatencyMode, TRUE}, + std::pair{CODECAPI_AVDecVideoMaxCodedWidth, 2560}, + std::pair{CODECAPI_AVDecVideoMaxCodedHeight, 1440}}) { + VARIANT value; + VariantInit(&value); + value.vt = VT_UI4; + value.ulVal = setting.second; + Check(codec->SetValue(&setting.first, &value), "codec-setting"); + } + } + + ~H264Decoder() { + if (configured_) (void)decoder_->ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0); + } + H264Decoder(const H264Decoder&) = delete; + H264Decoder& operator=(const H264Decoder&) = delete; + + void Decode(const InputEnvelope& input, const Consumer& consume) { + if (input.kind != 2 || input.data.empty() || input.data.size() > 4 * 1024 * 1024) + throw std::runtime_error("Invalid H264 decoder input"); + if (!configured_) { + if (!input.recovery) throw std::runtime_error("H264 decoder needs an independent recovery input"); + Microsoft::WRL::ComPtr type; + Check(MFCreateMediaType(&type), "input-type"); + Check(type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video), "input-major"); + Check(type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264), "input-subtype"); + Check(MFSetAttributeSize(type.Get(), MF_MT_FRAME_SIZE, input.width, input.height), "input-size"); + Check(type->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive), "input-progressive"); + Check(decoder_->SetInputType(0, type.Get(), 0), "set-input"); + SelectOutput(); + Check(decoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0), "begin"); + Check(decoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0), "start"); + configured_ = true; + } + Microsoft::WRL::ComPtr buffer; + Check(MFCreateMemoryBuffer(static_cast(input.data.size()), &buffer), "input-buffer"); + BYTE* bytes = nullptr; + Check(buffer->Lock(&bytes, nullptr, nullptr), "input-lock"); + std::copy(input.data.begin(), input.data.end(), bytes); + Check(buffer->Unlock(), "input-unlock"); + Check(buffer->SetCurrentLength(static_cast(input.data.size())), "input-length"); + Microsoft::WRL::ComPtr sample; + Check(MFCreateSample(&sample), "input-sample"); + Check(sample->AddBuffer(buffer.Get()), "input-sample-buffer"); + Check(sample->SetSampleTime(static_cast(input.timestamp)), "input-timestamp"); + Check(sample->SetSampleDuration(static_cast(input.duration)), "input-duration"); + HRESULT result = decoder_->ProcessInput(0, sample.Get(), 0); + if (result == MF_E_NOTACCEPTING) { + Drain(consume); + result = decoder_->ProcessInput(0, sample.Get(), 0); + } + Check(result, "process-input"); + Drain(consume); + } + + private: + static void Check(HRESULT result, const char* operation) { + if (FAILED(result)) throw std::runtime_error(std::string("H264 decoder ") + operation + " failed"); + } + + void SelectOutput() { + for (DWORD index = 0; index < 64; ++index) { + Microsoft::WRL::ComPtr type; + const HRESULT result = decoder_->GetOutputAvailableType(0, index, &type); + if (result == MF_E_NO_MORE_TYPES) break; + Check(result, "output-type"); + GUID subtype{}; + Check(type->GetGUID(MF_MT_SUBTYPE, &subtype), "output-subtype"); + if (subtype != MFVideoFormat_NV12) continue; + UINT32 width = 0, height = 0; + Check(MFGetAttributeSize(type.Get(), MF_MT_FRAME_SIZE, &width, &height), "output-size"); + if (width < 2 || height < 2 || width > 2560 || height > 1440 || (width & 1) || (height & 1)) { + throw std::runtime_error("H264 decoder output dimensions exceed the source bound"); + } + Check(decoder_->SetOutputType(0, type.Get(), 0), "set-output"); + UINT32 stride = 0; + if (FAILED(type->GetUINT32(MF_MT_DEFAULT_STRIDE, &stride))) { + LONG fallback = 0; + Check(MFGetStrideForBitmapInfoHeader(MFVideoFormat_NV12.Data1, width, &fallback), "output-stride"); + stride = static_cast(fallback); + } + if (static_cast(stride) < static_cast(width)) throw std::runtime_error("H264 decoder returned an invalid NV12 stride"); + width_ = width; + height_ = height; + stride_ = static_cast(stride); + return; + } + throw std::runtime_error("H264 decoder has no NV12 output"); + } + + void Drain(const Consumer& consume) { + for (unsigned iteration = 0; iteration < 64; ++iteration) { + MFT_OUTPUT_STREAM_INFO info{}; + Check(decoder_->GetOutputStreamInfo(0, &info), "output-info"); + Microsoft::WRL::ComPtr owned; + if (!(info.dwFlags & (MFT_OUTPUT_STREAM_PROVIDES_SAMPLES | MFT_OUTPUT_STREAM_CAN_PROVIDE_SAMPLES))) { + const DWORD expected = static_cast(stride_) * height_ * 3 / 2; + const DWORD size = std::max(info.cbSize, expected); + if (size > 2560 * 1440 * 2) throw std::runtime_error("H264 decoder output allocation exceeds its bound"); + Microsoft::WRL::ComPtr buffer; + Check(MFCreateAlignedMemoryBuffer(size, info.cbAlignment ? info.cbAlignment - 1 : 0, &buffer), "output-buffer"); + Check(MFCreateSample(&owned), "output-sample"); + Check(owned->AddBuffer(buffer.Get()), "output-sample-buffer"); + } + MFT_OUTPUT_DATA_BUFFER output{}; + output.pSample = owned.Get(); + DWORD status = 0; + const HRESULT result = decoder_->ProcessOutput(0, 1, &output, &status); + Microsoft::WRL::ComPtr events; + events.Attach(output.pEvents); + if (!owned) owned.Attach(output.pSample); + if (result == MF_E_TRANSFORM_NEED_MORE_INPUT) return; + if (result == MF_E_TRANSFORM_STREAM_CHANGE) { + SelectOutput(); + continue; + } + Check(result, "process-output"); + if (owned && !(output.dwStatus & MFT_OUTPUT_DATA_BUFFER_NO_SAMPLE)) { + consume(owned.Get(), width_, height_, stride_); + } + } + throw std::runtime_error("H264 decoder exceeded its bounded output pump"); + } + + Microsoft::WRL::ComPtr decoder_; + UINT32 width_ = 0; + UINT32 height_ = 0; + LONG stride_ = 0; + bool configured_ = false; +}; + +} // namespace screener::capture diff --git a/native/capture/windows/h264_encoder.cpp b/native/capture/windows/h264_encoder.cpp new file mode 100644 index 0000000..d829bbe --- /dev/null +++ b/native/capture/windows/h264_encoder.cpp @@ -0,0 +1,746 @@ +#include "h264_encoder.h" + +#include "capture_output.h" + +#include +#include +#include +#include + +#include +#include + +namespace screener::capture::windows { + +namespace { +constexpr DWORD kMaxEncodedSampleBytes = 4 * 1024 * 1024; +} + +[[noreturn]] void Fail(const std::string& stage, const std::string& detail) { + throw GateFailure(stage, detail); +} + +void Check(HRESULT result, const std::string& stage) { + if (FAILED(result)) { + throw GateFailure(stage, "Windows API returned a failing HRESULT", result); + } +} + +std::string NarrowAscii(const std::wstring& value) { + std::string result; + result.reserve(value.size()); + for (wchar_t character : value) { + result.push_back(character >= 32 && character <= 126 + ? static_cast(character) + : '?'); + } + return result; +} + +std::string GuidString(const GUID& value) { + wchar_t buffer[40] = {}; + if (StringFromGUID2(value, buffer, static_cast(std::size(buffer))) == 0) { + return "unavailable"; + } + return NarrowAscii(buffer); +} + +std::string HexByte(UINT8 value) { + std::ostringstream output; + output << std::hex << std::setfill('0') << std::setw(2) + << static_cast(value); + return output.str(); +} + +std::string ProfileLevelId(UINT8 profile, UINT8 constraints, UINT8 level) { + return HexByte(profile) + HexByte(constraints) + HexByte(level); +} + +std::vector EnumerateAdapters() { + ComPtr factory; + Check(CreateDXGIFactory1(IID_PPV_ARGS(&factory)), "dxgi-factory"); + + std::vector adapters; + for (UINT index = 0;; ++index) { + ComPtr adapter; + HRESULT result = factory->EnumAdapters1(index, &adapter); + if (result == DXGI_ERROR_NOT_FOUND) { + break; + } + Check(result, "dxgi-enumerate-adapter"); + + Adapter current; + current.index = index; + current.value = adapter; + Check(adapter->GetDesc1(¤t.description), "dxgi-adapter-description"); + if ((current.description.Flags & DXGI_ADAPTER_FLAG_SOFTWARE) == 0) { + adapters.push_back(std::move(current)); + } + } + return adapters; +} + +const Adapter& SelectAdapter(const std::vector& adapters, UINT index) { + auto found = std::find_if(adapters.begin(), adapters.end(), + [index](const Adapter& adapter) { + return adapter.index == index; + }); + if (found == adapters.end()) { + Fail("adapter-select", "selected DXGI adapter index is unavailable"); + } + return *found; +} + +DeviceContext CreateDevice(const Adapter& adapter) { + DeviceContext result; + constexpr D3D_FEATURE_LEVEL levels[] = { + D3D_FEATURE_LEVEL_12_1, D3D_FEATURE_LEVEL_12_0, + D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0, + }; + Check(D3D11CreateDevice(adapter.value.Get(), D3D_DRIVER_TYPE_UNKNOWN, nullptr, + D3D11_CREATE_DEVICE_VIDEO_SUPPORT | + D3D11_CREATE_DEVICE_BGRA_SUPPORT, + levels, static_cast(std::size(levels)), + D3D11_SDK_VERSION, &result.device, + &result.feature_level, &result.context), + "d3d11-create-device"); + + ComPtr multithread; + Check(result.device.As(&multithread), "d3d11-multithread-interface"); + multithread->SetMultithreadProtected(TRUE); + if (!multithread->GetMultithreadProtected()) { + Fail("d3d11-multithread-protection", + "D3D11 device did not enable multithread protection"); + } + + UINT reset_token = 0; + Check(MFCreateDXGIDeviceManager(&reset_token, &result.manager), + "dxgi-device-manager-create"); + Check(result.manager->ResetDevice(result.device.Get(), reset_token), + "dxgi-device-manager-reset"); + return result; +} + +std::wstring GetOptionalString(IMFAttributes* attributes, const GUID& key) { + wchar_t* raw = nullptr; + UINT32 length = 0; + HRESULT result = attributes->GetAllocatedString(key, &raw, &length); + if (result == MF_E_ATTRIBUTENOTFOUND) { + return L""; + } + Check(result, "mft-activation-string"); + std::wstring value(raw, length); + CoTaskMemFree(raw); + return value; +} + +ActivationList EnumerateHardwareEncoders(const Adapter& adapter, + bool require_candidate) { + ComPtr attributes; + Check(MFCreateAttributes(&attributes, 1), "mft-enumeration-attributes"); + Check(attributes->SetBlob(MFT_ENUM_ADAPTER_LUID, + reinterpret_cast( + &adapter.description.AdapterLuid), + sizeof(LUID)), + "mft-enumeration-adapter-luid"); + + MFT_REGISTER_TYPE_INFO input = {MFMediaType_Video, MFVideoFormat_NV12}; + MFT_REGISTER_TYPE_INFO output = {MFMediaType_Video, MFVideoFormat_H264}; + ActivationList list; + Check(MFTEnum2(MFT_CATEGORY_VIDEO_ENCODER, + MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_SORTANDFILTER, + &input, &output, attributes.Get(), &list.values, &list.count), + "mft-hardware-enumeration"); + if (require_candidate && list.count == 0) { + Fail("mft-hardware-enumeration", + "selected adapter exposed no hardware NV12-to-H264 MFT"); + } + return list; +} + +SelectedTransform ActivateTransform(const ActivationList& list, UINT index, + IMFDXGIDeviceManager* manager) { + if (index >= list.count) { + Fail("mft-select", "selected hardware MFT index is unavailable"); + } + + IMFActivate* activation = list.values[index]; + std::wstring friendly = GetOptionalString(activation, MFT_FRIENDLY_NAME_Attribute); + if (friendly.empty()) { + Fail("mft-identity", "hardware MFT has no friendly-name attribute"); + } + + GUID clsid = {}; + Check(activation->GetGUID(MFT_TRANSFORM_CLSID_Attribute, &clsid), + "mft-clsid"); + + PROPVARIANT hardware_url; + PropVariantInit(&hardware_url); + HRESULT hardware_result = activation->GetItem( + MFT_ENUM_HARDWARE_URL_Attribute, &hardware_url); + PropVariantClear(&hardware_url); + if (FAILED(hardware_result)) { + Fail("mft-hardware-identity", + "hardware-only activation omitted its hardware URL attribute"); + } + + SelectedTransform selected; + selected.activation = activation; + selected.name = NarrowAscii(friendly); + selected.clsid = GuidString(clsid); + Check(activation->ActivateObject(IID_PPV_ARGS(&selected.transform)), + "mft-activate"); + + ComPtr transform_attributes; + Check(selected.transform->GetAttributes(&transform_attributes), + "mft-global-attributes"); + UINT32 is_async = 0; + Check(transform_attributes->GetUINT32(MF_TRANSFORM_ASYNC, &is_async), + "mft-async-readback"); + if (is_async != TRUE) { + Fail("mft-async-readback", "hardware MFT is not asynchronous"); + } + Check(transform_attributes->SetUINT32(MF_TRANSFORM_ASYNC_UNLOCK, TRUE), + "mft-async-unlock"); + + UINT32 d3d11_aware = 0; + Check(transform_attributes->GetUINT32(MF_SA_D3D11_AWARE, &d3d11_aware), + "mft-d3d11-aware-readback"); + if (d3d11_aware != TRUE) { + Fail("mft-d3d11-aware-readback", "hardware MFT is not D3D11-aware"); + } + + Check(selected.transform->ProcessMessage( + MFT_MESSAGE_SET_D3D_MANAGER, + reinterpret_cast(manager)), + "mft-set-d3d-manager"); + Check(selected.transform.As(&selected.codec), "mft-codec-api"); + Check(selected.transform.As(&selected.events), "mft-event-generator"); + return selected; +} + +void RequireProperty(ICodecAPI* codec, const GUID& key, + const std::string& name) { + HRESULT supported = codec->IsSupported(&key); + if (supported != S_OK) { + if (FAILED(supported) && supported != E_NOTIMPL) { + Check(supported, name + "-supported-query"); + } + Fail(name + "-unsupported", "required codec property is unsupported"); + } + HRESULT modifiable = codec->IsModifiable(&key); + if (modifiable == S_FALSE) { + Fail(name + "-readonly", "required codec property is read-only"); + } + // Some hardware MFTs do not implement this advisory query. SetValue=S_OK + // followed by exact GetValue readback remains the authoritative gate. + if (modifiable != S_OK && modifiable != E_NOTIMPL) { + Check(modifiable, name + "-modifiable-query"); + } +} + +UINT32 ReadU32(ICodecAPI* codec, const GUID& key, const std::string& name) { + VARIANT value; + VariantInit(&value); + Check(codec->GetValue(&key, &value), name + "-readback"); + if (value.vt != VT_UI4) { + VariantClear(&value); + Fail(name + "-readback-type", "codec property did not return VT_UI4"); + } + UINT32 result = value.ulVal; + VariantClear(&value); + return result; +} + +bool ReadBool(ICodecAPI* codec, const GUID& key, const std::string& name) { + VARIANT value; + VariantInit(&value); + Check(codec->GetValue(&key, &value), name + "-readback"); + if (value.vt != VT_BOOL) { + VariantClear(&value); + Fail(name + "-readback-type", "codec property did not return VT_BOOL"); + } + bool result = value.boolVal != VARIANT_FALSE; + VariantClear(&value); + return result; +} + +void SetU32(ICodecAPI* codec, const GUID& key, UINT32 expected, + const std::string& name) { + RequireProperty(codec, key, name); + VARIANT value; + VariantInit(&value); + value.vt = VT_UI4; + value.ulVal = expected; + HRESULT set_result = codec->SetValue(&key, &value); + VariantClear(&value); + if (set_result != S_OK) { + if (FAILED(set_result)) { + Check(set_result, name + "-set"); + } + Fail(name + "-readonly", "codec property rejected the requested value"); + } + if (ReadU32(codec, key, name) != expected) { + Fail(name + "-weakened", "codec property readback changed the requested value"); + } +} + +void SetBool(ICodecAPI* codec, const GUID& key, bool expected, + const std::string& name) { + RequireProperty(codec, key, name); + VARIANT value; + VariantInit(&value); + value.vt = VT_BOOL; + value.boolVal = expected ? VARIANT_TRUE : VARIANT_FALSE; + HRESULT set_result = codec->SetValue(&key, &value); + VariantClear(&value); + if (set_result != S_OK) { + if (FAILED(set_result)) { + Check(set_result, name + "-set"); + } + Fail(name + "-readonly", "codec property rejected the requested value"); + } + if (ReadBool(codec, key, name) != expected) { + Fail(name + "-weakened", "codec property readback changed the requested value"); + } +} + +void SetOptionalU32(ICodecAPI* codec, const GUID& key, UINT32 value) { + if (codec->IsSupported(&key) != S_OK) return; + VARIANT requested; + VariantInit(&requested); + requested.vt = VT_UI4; + requested.ulVal = value; + (void)codec->SetValue(&key, &requested); + VariantClear(&requested); +} + +void ConfigureCodec( + ICodecAPI* codec, + const VideoProfile& profile) { + SetU32(codec, CODECAPI_AVEncCommonRateControlMode, + eAVEncCommonRateControlMode_CBR, "codec-cbr"); + SetBool(codec, CODECAPI_AVLowLatencyMode, true, "codec-low-latency"); + SetU32(codec, CODECAPI_AVEncCommonMeanBitRate, profile.bit_rate, + "codec-mean-bitrate"); + SetU32(codec, CODECAPI_AVEncMPVGOPSize, profile.gop_frames(), "codec-gop"); + SetOptionalU32(codec, CODECAPI_AVEncCommonQualityVsSpeed, + profile.quality_vs_speed()); + RequireProperty(codec, CODECAPI_AVEncVideoForceKeyFrame, + "codec-force-keyframe"); +} + +ComPtr CreateOutputType( + const VideoProfile& profile) { + ComPtr type; + Check(MFCreateMediaType(&type), "output-type-create"); + Check(type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video), + "output-type-major"); + Check(type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264), + "output-type-subtype"); + Check(type->SetUINT32(MF_MT_AVG_BITRATE, profile.bit_rate), + "output-type-bitrate"); + Check(MFSetAttributeSize(type.Get(), MF_MT_FRAME_SIZE, profile.width, + profile.height), + "output-type-size"); + Check(MFSetAttributeRatio(type.Get(), MF_MT_FRAME_RATE, profile.frame_rate, 1), + "output-type-framerate"); + Check(MFSetAttributeRatio(type.Get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1), + "output-type-pixel-aspect"); + Check(type->SetUINT32(MF_MT_INTERLACE_MODE, + MFVideoInterlace_Progressive), + "output-type-interlace"); + Check(type->SetUINT32(MF_MT_MPEG2_PROFILE, eAVEncH264VProfile_Base), + "output-type-profile"); + Check(type->SetUINT32(MF_MT_MPEG2_LEVEL, profile.h264_level()), + "output-type-level"); + return type; +} + +ComPtr CreateInputType( + const VideoProfile& profile) { + ComPtr type; + Check(MFCreateMediaType(&type), "input-type-create"); + Check(type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video), + "input-type-major"); + Check(type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12), + "input-type-subtype"); + Check(MFSetAttributeSize(type.Get(), MF_MT_FRAME_SIZE, profile.width, + profile.height), + "input-type-size"); + Check(MFSetAttributeRatio(type.Get(), MF_MT_FRAME_RATE, profile.frame_rate, 1), + "input-type-framerate"); + Check(MFSetAttributeRatio(type.Get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1), + "input-type-pixel-aspect"); + Check(type->SetUINT32(MF_MT_INTERLACE_MODE, + MFVideoInterlace_Progressive), + "input-type-interlace"); + Check(type->SetUINT32(MF_MT_DEFAULT_STRIDE, profile.width), + "input-type-stride"); + return type; +} + +void RequireTypeU32(IMFAttributes* type, const GUID& key, UINT32 expected, + const std::string& stage) { + UINT32 actual = 0; + Check(type->GetUINT32(key, &actual), stage + "-readback"); + if (actual != expected) { + Fail(stage + "-weakened", "media type changed a required value"); + } +} + +void ValidateMediaTypes( + IMFTransform* transform, + const VideoProfile& profile) { + ComPtr output; + Check(transform->GetOutputCurrentType(0, &output), + "output-type-current"); + GUID subtype = {}; + Check(output->GetGUID(MF_MT_SUBTYPE, &subtype), "output-type-subtype-readback"); + if (subtype != MFVideoFormat_H264) { + Fail("output-type-subtype-weakened", "output is not H264"); + } + RequireTypeU32(output.Get(), MF_MT_AVG_BITRATE, profile.bit_rate, + "output-type-bitrate"); + RequireTypeU32(output.Get(), MF_MT_MPEG2_PROFILE, + eAVEncH264VProfile_Base, "output-type-profile"); + RequireTypeU32(output.Get(), MF_MT_MPEG2_LEVEL, profile.h264_level(), + "output-type-level"); + + UINT32 width = 0; + UINT32 height = 0; + Check(MFGetAttributeSize(output.Get(), MF_MT_FRAME_SIZE, &width, &height), + "output-type-size-readback"); + if (width != profile.width || height != profile.height) { + Fail("output-type-size-weakened", "output dimensions changed"); + } + UINT32 numerator = 0; + UINT32 denominator = 0; + Check(MFGetAttributeRatio(output.Get(), MF_MT_FRAME_RATE, &numerator, + &denominator), + "output-type-framerate-readback"); + if (numerator != profile.frame_rate || denominator != 1) { + Fail("output-type-framerate-weakened", "output frame rate changed"); + } +} + +void ValidateCodecReadback( + ICodecAPI* codec, + const VideoProfile& profile) { + if (ReadU32(codec, CODECAPI_AVEncCommonRateControlMode, "codec-cbr") != + eAVEncCommonRateControlMode_CBR || + !ReadBool(codec, CODECAPI_AVLowLatencyMode, "codec-low-latency") || + ReadU32(codec, CODECAPI_AVEncCommonMeanBitRate, + "codec-mean-bitrate") != profile.bit_rate || + ReadU32(codec, CODECAPI_AVEncMPVGOPSize, "codec-gop") != + profile.gop_frames()) { + Fail("codec-final-readback", "one or more codec properties were weakened"); + } +} + +std::optional> FindStartCode( + const std::vector& bytes, size_t from) { + for (size_t index = from; index + 3 <= bytes.size(); ++index) { + if (bytes[index] != 0 || bytes[index + 1] != 0) { + continue; + } + if (bytes[index + 2] == 1) { + return std::pair{index, 3}; + } + if (index + 4 <= bytes.size() && bytes[index + 2] == 0 && + bytes[index + 3] == 1) { + return std::pair{index, 4}; + } + } + return std::nullopt; +} + +NalSummary InspectAnnexB(const std::vector& bytes) { + NalSummary summary; + std::optional> current = FindStartCode(bytes, 0); + if (!current || current->first != 0) { + return summary; + } + summary.annex_b = true; + while (current) { + size_t nal_start = current->first + current->second; + std::optional> next = FindStartCode(bytes, nal_start); + size_t nal_end = next ? next->first : bytes.size(); + if (nal_start < nal_end) { + UINT8 nal_type = bytes[nal_start] & 0x1f; + summary.sps = summary.sps || nal_type == 7; + summary.pps = summary.pps || nal_type == 8; + summary.idr = summary.idr || nal_type == 5; + if (nal_type == 7 && nal_end - nal_start >= 4) { + std::string value = ProfileLevelId(bytes[nal_start + 1], + bytes[nal_start + 2], + bytes[nal_start + 3]); + if (summary.profile_level_id && *summary.profile_level_id != value) { + Fail("bitstream-sps-consistency", + "one access unit contained conflicting SPS profile bytes"); + } + summary.profile_level_id = value; + } + } + current = next; + } + return summary; +} + +std::vector ReadSample(IMFSample* sample) { + DWORD total_length = 0; + Check(sample->GetTotalLength(&total_length), "output-total-length"); + if (total_length == 0 || total_length > kMaxEncodedSampleBytes) { + Fail("output-buffer-bounds", "encoded output size is empty or unbounded"); + } + + ComPtr buffer; + Check(sample->ConvertToContiguousBuffer(&buffer), "output-contiguous-buffer"); + DWORD length = 0; + Check(buffer->GetCurrentLength(&length), "output-buffer-length"); + if (length != total_length) { + Fail("output-buffer-length", "contiguous output length changed unexpectedly"); + } + std::vector bytes(length); + UINT8* raw = nullptr; + DWORD maximum = 0; + DWORD current = 0; + Check(buffer->Lock(&raw, &maximum, ¤t), "output-buffer-lock"); + if (raw == nullptr || current != length || current > kMaxEncodedSampleBytes) { + buffer->Unlock(); + Fail("output-buffer-lock-bounds", + "locked output length is inconsistent or unbounded"); + } + std::copy(raw, raw + current, bytes.begin()); + Check(buffer->Unlock(), "output-buffer-unlock"); + return bytes; +} + +void ForceKeyFrame(ICodecAPI* codec) { + VARIANT value; + VariantInit(&value); + value.vt = VT_UI4; + value.ulVal = 1; + HRESULT set_result = + codec->SetValue(&CODECAPI_AVEncVideoForceKeyFrame, &value); + VariantClear(&value); + if (set_result != S_OK) { + if (FAILED(set_result)) { + Check(set_result, "codec-force-keyframe-set"); + } + Fail("codec-force-keyframe-readonly", + "force-key-frame command was not accepted"); + } +} + +ComPtr CreateCallerOutputSample(const MFT_OUTPUT_STREAM_INFO& info) { + if (info.cbSize > kMaxEncodedSampleBytes) { + Fail("output-buffer-contract", + "hardware MFT requested an output buffer larger than four MiB"); + } + ComPtr sample; + Check(MFCreateSample(&sample), "output-sample-create"); + ComPtr buffer; + Check(MFCreateMemoryBuffer(kMaxEncodedSampleBytes, &buffer), + "output-memory-buffer"); + Check(sample->AddBuffer(buffer.Get()), "output-sample-buffer"); + return sample; +} + +ComPtr PullOutput(IMFTransform* transform, + const MFT_OUTPUT_STREAM_INFO& info) { + bool transform_provides = + (info.dwFlags & (MFT_OUTPUT_STREAM_PROVIDES_SAMPLES | + MFT_OUTPUT_STREAM_CAN_PROVIDE_SAMPLES)) != 0; + ComPtr caller_sample; + if (!transform_provides) { + caller_sample = CreateCallerOutputSample(info); + } + + MFT_OUTPUT_DATA_BUFFER output = {}; + output.dwStreamID = 0; + output.pSample = caller_sample.Get(); + DWORD status = 0; + HRESULT result = transform->ProcessOutput(0, 1, &output, &status); + + ComPtr events; + if (output.pEvents != nullptr) { + events.Attach(output.pEvents); + } + Check(result, "mft-process-output"); + if ((output.dwStatus & MFT_OUTPUT_DATA_BUFFER_FORMAT_CHANGE) != 0 || + (status & MFT_PROCESS_OUTPUT_STATUS_NEW_STREAMS) != 0) { + Fail("mft-output-format-change", + "hardware MFT changed format during the fixed run"); + } + + if (caller_sample) { + if (output.pSample != caller_sample.Get()) { + if (output.pSample != nullptr) { + output.pSample->Release(); + } + Fail("mft-output-allocation", + "hardware MFT replaced the caller-owned output sample"); + } + return caller_sample; + } + if (output.pSample == nullptr) { + Fail("mft-output-sample", "hardware MFT produced no output sample"); + } + ComPtr produced; + produced.Attach(output.pSample); + return produced; +} + +ComPtr CreateSurfaceSample(ID3D11Texture2D* texture, + LONGLONG timestamp100ns, + LONGLONG duration100ns) { + ComPtr buffer; + Check(MFCreateDXGISurfaceBuffer(IID_ID3D11Texture2D, texture, 0, FALSE, + &buffer), + "input-dxgi-buffer"); + ComPtr sample; + Check(MFCreateVideoSampleFromSurface(nullptr, &sample), + "input-video-sample"); + Check(sample->AddBuffer(buffer.Get()), "input-sample-buffer"); + Check(sample->SetSampleTime(timestamp100ns), "input-sample-time"); + Check(sample->SetSampleDuration(duration100ns), "input-sample-duration"); + return sample; +} + +void RequireEncoderTime(EncoderClock::time_point deadline) { + if (EncoderClock::now() >= deadline) { + Fail("codec-probe-timeout", "native codec selection exceeded its time budget"); + } +} + +LiveEncoder::LiveEncoder( + SelectedTransform selected, + VideoProfile profile) + : VideoEncoder(OutputKind::h264, selected.name, selected.clsid), + selected_(std::move(selected)), profile_(profile) { + ConfigureCodec(selected_.codec.Get(), profile_); + ComPtr output_type = CreateOutputType(profile_); + Check(selected_.transform->SetOutputType(0, output_type.Get(), 0), + "mft-set-output-type"); + ComPtr input_type = CreateInputType(profile_); + Check(selected_.transform->SetInputType(0, input_type.Get(), 0), + "mft-set-input-type"); + ValidateMediaTypes(selected_.transform.Get(), profile_); + ValidateCodecReadback(selected_.codec.Get(), profile_); + Check(selected_.transform->GetOutputStreamInfo(0, &output_info_), + "mft-output-stream-info"); + Check(selected_.transform->ProcessMessage( + MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0), + "mft-begin-streaming"); + Check(selected_.transform->ProcessMessage( + MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0), + "mft-start-stream"); + started_ = true; +} + +LiveEncoder::~LiveEncoder() { + if (started_) { + selected_.transform->ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0); + selected_.transform->ProcessMessage(MFT_MESSAGE_NOTIFY_END_STREAMING, 0); + } +} + +void LiveEncoder::SetBitrate(UINT32 bitrate) { + SetU32(selected_.codec.Get(), CODECAPI_AVEncCommonMeanBitRate, bitrate, + "codec-live-bitrate"); + profile_.bit_rate = bitrate; +} + +EncodedAccessUnit LiveEncoder::Encode( + ID3D11Texture2D* texture, UINT64 timestamp100ns, bool force_key_frame, + EncoderClock::time_point probe_deadline) { + WaitForInput(probe_deadline); + if (force_key_frame) ForceKeyFrame(selected_.codec.Get()); + ComPtr sample = CreateSurfaceSample( + texture, static_cast(timestamp100ns), + profile_.frame_duration_100ns()); + Check(selected_.transform->ProcessInput(0, sample.Get(), 0), + "mft-process-input"); + --input_requests_; + + const auto deadline = std::min(probe_deadline, + EncoderClock::now() + std::chrono::seconds(2)); + while (std::chrono::steady_clock::now() < deadline) { + MediaEventType type = NextEvent(deadline); + if (type == METransformNeedInput) { + ++input_requests_; + continue; + } + if (type != METransformHaveOutput) continue; + + ComPtr output = PullOutput(selected_.transform.Get(), + output_info_); + LONGLONG output_time = -1; + Check(output->GetSampleTime(&output_time), "output-sample-time"); + if (output_time != static_cast(timestamp100ns)) { + Fail("output-order", "hardware MFT changed or reordered a live timestamp"); + } + std::vector bytes = ReadSample(output.Get()); + NalSummary nal = InspectAnnexB(bytes); + if (!nal.annex_b) { + Fail("bitstream-annexb", "live output is not Annex-B H264"); + } + if (nal.profile_level_id) { + if (*nal.profile_level_id != profile_.profile_level_id()) { + Fail("bitstream-profile", "hardware MFT changed the requested H.264 profile level"); + } + if (profile_level_id_ && *profile_level_id_ != *nal.profile_level_id) { + Fail("bitstream-profile-change", "hardware MFT changed SPS profile"); + } + profile_level_id_ = nal.profile_level_id; + } + if (force_key_frame && !(nal.sps && nal.pps && nal.idr)) { + Fail("bitstream-recovery-unit", + "requested live recovery unit lacks SPS, PPS, or IDR"); + } + if (!profile_level_id_) { + Fail("bitstream-sps", "first live access unit did not carry an SPS"); + } + RequireEncoderTime(probe_deadline); + return EncodedAccessUnit{timestamp100ns, nal.idr && nal.sps && nal.pps, + std::move(bytes)}; + } + Fail("mft-output-timeout", "hardware MFT did not produce live output in time"); +} + +MediaEventType LiveEncoder::NextEvent(std::chrono::steady_clock::time_point deadline) { + while (std::chrono::steady_clock::now() < deadline) { + ComPtr event; + HRESULT result = selected_.events->GetEvent(MF_EVENT_FLAG_NO_WAIT, + &event); + if (result == MF_E_NO_EVENTS_AVAILABLE) { + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + continue; + } + Check(result, "mft-get-event"); + HRESULT status = S_OK; + Check(event->GetStatus(&status), "mft-event-status-read"); + Check(status, "mft-event-status"); + MediaEventType type = MEUnknown; + Check(event->GetType(&type), "mft-event-type"); + if (type == MEError) Fail("mft-error-event", "hardware MFT emitted MEError"); + return type; + } + Fail("mft-event-timeout", "hardware MFT did not request input in time"); +} + +void LiveEncoder::WaitForInput(EncoderClock::time_point probe_deadline) { + const auto deadline = std::min(probe_deadline, + EncoderClock::now() + std::chrono::seconds(2)); + while (input_requests_ == 0) { + MediaEventType type = NextEvent(deadline); + if (type == METransformNeedInput) { + ++input_requests_; + } else if (type == METransformHaveOutput) { + Fail("mft-unexpected-output", "hardware MFT produced output without live input"); + } + } +} + +} // namespace screener::capture::windows diff --git a/native/capture/windows/h264_encoder.h b/native/capture/windows/h264_encoder.h new file mode 100644 index 0000000..e658b4d --- /dev/null +++ b/native/capture/windows/h264_encoder.h @@ -0,0 +1,253 @@ +#pragma once + +#ifndef NOMINMAX +#define NOMINMAX +#endif +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace screener::capture { +enum class OutputKind : UINT8; +} + +namespace screener::capture::windows { + +using Microsoft::WRL::ComPtr; + +enum class DegradationPreference { + resolution, + balanced, + framerate, +}; + +struct VideoProfile final { + UINT32 width = 1280; + UINT32 height = 720; + UINT32 frame_rate = 30; + UINT32 bit_rate = 3'000'000; + DegradationPreference preference = DegradationPreference::balanced; + + UINT32 gop_frames() const { return frame_rate * 2; } + LONGLONG frame_duration_100ns() const { + return 10'000'000 / static_cast(frame_rate); + } + UINT32 quality_vs_speed() const { + if (preference == DegradationPreference::resolution) return 100; + if (preference == DegradationPreference::framerate) return 0; + return 50; + } + UINT32 h264_level() const { + const UINT64 macroblocks = + ((static_cast(width) + 15) / 16) * + ((static_cast(height) + 15) / 16); + const UINT64 macroblocks_per_second = macroblocks * frame_rate; + if (macroblocks <= 3'600 && macroblocks_per_second <= 108'000) return 31; + if (macroblocks <= 8'192 && macroblocks_per_second <= 245'760) return 40; + if (macroblocks <= 8'704 && macroblocks_per_second <= 522'240) return 42; + if (macroblocks <= 22'080 && macroblocks_per_second <= 589'824) return 50; + return 51; + } + std::string profile_level_id() const { + std::ostringstream output; + output << "42c0" << std::hex << std::setfill('0') << std::setw(2) + << h264_level(); + return output.str(); + } +}; + +constexpr VideoProfile kDefaultVideoProfile{}; + +class GateFailure final : public std::runtime_error { + public: + GateFailure(std::string stage, std::string detail, HRESULT result = S_OK) + : std::runtime_error(detail), stage_(std::move(stage)), result_(result) {} + + const std::string& stage() const noexcept { return stage_; } + HRESULT result() const noexcept { return result_; } + + private: + std::string stage_; + HRESULT result_; +}; + +[[noreturn]] void Fail(const std::string& stage, const std::string& detail); +void Check(HRESULT result, const std::string& stage); + +struct Runtime final { + Runtime() { + Check(CoInitializeEx(nullptr, COINIT_MULTITHREADED), "com-initialize"); + com_started = true; + Check(MFStartup(MF_VERSION, MFSTARTUP_FULL), "mf-startup"); + mf_started = true; + } + + ~Runtime() { + if (mf_started) { + MFShutdown(); + } + if (com_started) { + CoUninitialize(); + } + } + + bool com_started = false; + bool mf_started = false; +}; + +struct Adapter final { + UINT index = 0; + DXGI_ADAPTER_DESC1 description = {}; + ComPtr value; +}; + +struct DeviceContext final { + ComPtr device; + ComPtr context; + ComPtr manager; + D3D_FEATURE_LEVEL feature_level = D3D_FEATURE_LEVEL_9_1; +}; + +class ActivationList final { + public: + ActivationList() = default; + ActivationList(const ActivationList&) = delete; + ActivationList& operator=(const ActivationList&) = delete; + ActivationList(ActivationList&& other) noexcept + : values(std::exchange(other.values, nullptr)), + count(std::exchange(other.count, 0)) {} + + ~ActivationList() { + for (UINT32 index = 0; index < count; ++index) { + values[index]->Release(); + } + CoTaskMemFree(values); + } + + IMFActivate** values = nullptr; + UINT32 count = 0; +}; + +struct SelectedTransform final { + SelectedTransform() = default; + SelectedTransform(const SelectedTransform&) = delete; + SelectedTransform& operator=(const SelectedTransform&) = delete; + SelectedTransform(SelectedTransform&&) noexcept = default; + SelectedTransform& operator=(SelectedTransform&&) = delete; + ~SelectedTransform() { if (activation) activation->ShutdownObject(); } + + std::string name; + std::string clsid; + ComPtr activation; + ComPtr transform; + ComPtr codec; + ComPtr events; +}; + +struct NalSummary final { + bool annex_b = false; + bool sps = false; + bool pps = false; + bool idr = false; + std::optional profile_level_id; +}; + +std::vector EnumerateAdapters(); +const Adapter& SelectAdapter(const std::vector& adapters, UINT index); +DeviceContext CreateDevice(const Adapter& adapter); +ActivationList EnumerateHardwareEncoders(const Adapter& adapter, + bool require_candidate = true); +SelectedTransform ActivateTransform(const ActivationList& list, UINT index, + IMFDXGIDeviceManager* manager); +std::wstring GetOptionalString(IMFAttributes* attributes, const GUID& key); +std::string NarrowAscii(const std::wstring& value); +std::string GuidString(const GUID& value); + +// The hardware fixture uses the same configuration and output validation. +void SetU32(ICodecAPI* codec, const GUID& key, UINT32 expected, + const std::string& name); +void ConfigureCodec(ICodecAPI* codec, + const VideoProfile& profile = kDefaultVideoProfile); +ComPtr CreateOutputType( + const VideoProfile& profile = kDefaultVideoProfile); +ComPtr CreateInputType( + const VideoProfile& profile = kDefaultVideoProfile); +void ValidateMediaTypes(IMFTransform* transform, + const VideoProfile& profile = kDefaultVideoProfile); +void ValidateCodecReadback(ICodecAPI* codec, + const VideoProfile& profile = kDefaultVideoProfile); +void ForceKeyFrame(ICodecAPI* codec); +ComPtr PullOutput(IMFTransform* transform, + const MFT_OUTPUT_STREAM_INFO& info); +NalSummary InspectAnnexB(const std::vector& bytes); +std::vector ReadSample(IMFSample* sample); + +struct EncodedAccessUnit final { + UINT64 timestamp100ns = 0; + bool key_frame = false; + std::vector bytes; +}; + +using EncoderClock = std::chrono::steady_clock; + +void RequireEncoderTime(EncoderClock::time_point deadline); + +class VideoEncoder { + public: + VideoEncoder(OutputKind kind, std::string name, std::string identity) + : kind(kind), name(std::move(name)), identity(std::move(identity)) {} + virtual ~VideoEncoder() = default; + virtual EncodedAccessUnit Encode(ID3D11Texture2D* texture, + UINT64 timestamp100ns, + bool force_key_frame, + EncoderClock::time_point deadline = + EncoderClock::time_point::max()) = 0; + virtual void SetBitrate(UINT32 bitrate) = 0; + const OutputKind kind; + const std::string name; + const std::string identity; +}; + +// The caller owns COM/MF startup and serializes access to each encoder. +class LiveEncoder final : public VideoEncoder { + public: + explicit LiveEncoder(SelectedTransform selected, + VideoProfile profile = kDefaultVideoProfile); + ~LiveEncoder() override; + LiveEncoder(const LiveEncoder&) = delete; + LiveEncoder& operator=(const LiveEncoder&) = delete; + + void SetBitrate(UINT32 bitrate) override; + EncodedAccessUnit Encode( + ID3D11Texture2D* texture, UINT64 timestamp100ns, bool force_key_frame, + EncoderClock::time_point deadline = EncoderClock::time_point::max()) override; + + private: + MediaEventType NextEvent(EncoderClock::time_point deadline); + void WaitForInput(EncoderClock::time_point deadline); + + SelectedTransform selected_; + VideoProfile profile_; + MFT_OUTPUT_STREAM_INFO output_info_ = {}; + UINT32 input_requests_ = 0; + std::optional profile_level_id_; + bool started_ = false; +}; + +} // namespace screener::capture::windows diff --git a/native/capture/windows/libvpx-dependencies.json b/native/capture/windows/libvpx-dependencies.json deleted file mode 100644 index 38c0899..0000000 --- a/native/capture/windows/libvpx-dependencies.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "libvpx": { - "version": "1.17.0", - "commit": "6df3ec34557879fff673706f4a1d9fbd0f3a6f0e", - "url": "https://github.com/webmproject/libvpx/archive/6df3ec34557879fff673706f4a1d9fbd0f3a6f0e.tar.gz", - "sha256": "073bd6202e4c55c8cb759abbb123aecd4cc8d99f52ba27af790178cfb1108423" - }, - "nasm": { - "version": "2.16.03", - "url": "https://www.nasm.us/pub/nasm/releasebuilds/2.16.03/win64/nasm-2.16.03-win64.zip", - "sha256": "3ee4782247bcb874378d02f7eab4e294a84d3d15f3f6ee2de2f47a46aa7226e6" - }, - "make": { - "url": "https://repo.msys2.org/msys/x86_64/make-4.4.1-2-x86_64.pkg.tar.zst", - "sha256": "2408af61717dae87b00c855b132769a125c708907fc94a46bb16dae076113e5c" - } -} diff --git a/native/capture/windows/main.cpp b/native/capture/windows/main.cpp index 0b856b2..d8d0b42 100644 --- a/native/capture/windows/main.cpp +++ b/native/capture/windows/main.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #ifdef SCREENER_H264_FIXTURE #include #include @@ -31,8 +32,16 @@ #include "capture_target.h" #include "capture_geometry.h" #include "process_audio.h" +#include "h264_encoder.h" #ifndef SCREENER_H264_FIXTURE -#include "vp8_encoder.h" +#include "adaptive_encoder.h" +#include "capture_control.h" +#include "capture_input.h" +#include "h264_decoder.h" +#include "capture_output.h" +#include "output_mailbox.h" +#include "vp8_decoder.h" +#include #endif #include @@ -76,92 +85,22 @@ IGraphicsCaptureSession5 : public IInspectable { namespace { -constexpr UINT32 kWidth = 1280; -constexpr UINT32 kHeight = 720; -constexpr UINT32 kFrameRate = 30; -constexpr UINT32 kBitRate = 3'000'000; +using namespace screener::capture::windows; + constexpr LONGLONG kFastCaptureUpdateInterval100ns = 40'000; -constexpr UINT32 kVbvBytes = kBitRate / kFrameRate / 8; -constexpr UINT32 kGopFrames = 60; -constexpr UINT32 kMaxEventsPerPump = 64; -constexpr DWORD kMaxEncodedSampleBytes = 4 * 1024 * 1024; #ifdef SCREENER_H264_FIXTURE +constexpr UINT32 kFrameRate = kDefaultVideoProfile.frame_rate; +constexpr UINT32 kBitRate = kDefaultVideoProfile.bit_rate; +constexpr UINT32 kGopFrames = kFrameRate * 2; +constexpr UINT32 kMaxEventsPerPump = 64; constexpr UINT32 kFrameCount = 360; constexpr UINT32 kRateProbeBitRate = kBitRate / 2; constexpr UINT32 kRateProbeFrames = kFrameCount / 3; constexpr UINT32 kMaxInFlight = 8; constexpr DWORD kMaxPdhArrayBytes = 4 * 1024 * 1024; constexpr DWORD kMaxPdhArrayItems = 16 * 1024; -#endif - constexpr LONGLONG kFrameDuration100ns = 10'000'000 / kFrameRate; - -enum class DegradationPreference { - resolution, - balanced, - framerate, -}; - -struct VideoProfile final { - UINT32 width = kWidth; - UINT32 height = kHeight; - UINT32 frame_rate = kFrameRate; - UINT32 bit_rate = kBitRate; - DegradationPreference preference = DegradationPreference::balanced; - - UINT32 vbv_bytes() const { return bit_rate / frame_rate / 8; } - UINT32 gop_frames() const { return frame_rate * 2; } - LONGLONG frame_duration_100ns() const { - return 10'000'000 / static_cast(frame_rate); - } - UINT32 quality_vs_speed() const { - if (preference == DegradationPreference::resolution) return 100; - if (preference == DegradationPreference::framerate) return 0; - return 50; - } - UINT32 h264_level() const { - const UINT64 macroblocks = - ((static_cast(width) + 15) / 16) * - ((static_cast(height) + 15) / 16); - const UINT64 macroblocks_per_second = macroblocks * frame_rate; - if (macroblocks <= 3'600 && macroblocks_per_second <= 108'000) return 31; - if (macroblocks <= 8'192 && macroblocks_per_second <= 245'760) return 40; - if (macroblocks <= 8'704 && macroblocks_per_second <= 522'240) return 42; - if (macroblocks <= 22'080 && macroblocks_per_second <= 589'824) return 50; - return 51; - } - std::string profile_level_id() const { - std::ostringstream output; - output << "42c0" << std::hex << std::setfill('0') << std::setw(2) - << h264_level(); - return output.str(); - } -}; - -constexpr VideoProfile kDefaultVideoProfile{}; - -class GateFailure final : public std::runtime_error { - public: - GateFailure(std::string stage, std::string detail, HRESULT result = S_OK) - : std::runtime_error(detail), stage_(std::move(stage)), result_(result) {} - - const std::string& stage() const noexcept { return stage_; } - HRESULT result() const noexcept { return result_; } - - private: - std::string stage_; - HRESULT result_; -}; - -[[noreturn]] void Fail(const std::string& stage, const std::string& detail) { - throw GateFailure(stage, detail); -} - -void Check(HRESULT result, const std::string& stage) { - if (FAILED(result)) { - throw GateFailure(stage, "Windows API returned a failing HRESULT", result); - } -} +#endif void EnableFastCaptureUpdates( const winrt::Windows::Graphics::Capture::GraphicsCaptureSession& session) { @@ -181,25 +120,6 @@ std::wstring Lower(std::wstring value) { return value; } -std::string NarrowAscii(const std::wstring& value) { - std::string result; - result.reserve(value.size()); - for (wchar_t character : value) { - result.push_back(character >= 32 && character <= 126 - ? static_cast(character) - : '?'); - } - return result; -} - -std::string GuidString(const GUID& value) { - wchar_t buffer[40] = {}; - if (StringFromGUID2(value, buffer, static_cast(std::size(buffer))) == 0) { - return "unavailable"; - } - return NarrowAscii(buffer); -} - std::string LuidString(const LUID& luid) { std::ostringstream output; output << "0x" << std::hex << std::setfill('0') << std::setw(8) @@ -208,68 +128,6 @@ std::string LuidString(const LUID& luid) { return output.str(); } -std::string HexByte(UINT8 value) { - std::ostringstream output; - output << std::hex << std::setfill('0') << std::setw(2) - << static_cast(value); - return output.str(); -} - -std::string ProfileLevelId(UINT8 profile, UINT8 constraints, UINT8 level) { - return HexByte(profile) + HexByte(constraints) + HexByte(level); -} - -struct Runtime final { - Runtime() { - Check(CoInitializeEx(nullptr, COINIT_MULTITHREADED), "com-initialize"); - com_started = true; - Check(MFStartup(MF_VERSION, MFSTARTUP_FULL), "mf-startup"); - mf_started = true; - } - - ~Runtime() { - if (mf_started) { - MFShutdown(); - } - if (com_started) { - CoUninitialize(); - } - } - - bool com_started = false; - bool mf_started = false; -}; - -struct Adapter final { - UINT index = 0; - DXGI_ADAPTER_DESC1 description = {}; - ComPtr value; -}; - -std::vector EnumerateAdapters() { - ComPtr factory; - Check(CreateDXGIFactory1(IID_PPV_ARGS(&factory)), "dxgi-factory"); - - std::vector adapters; - for (UINT index = 0;; ++index) { - ComPtr adapter; - HRESULT result = factory->EnumAdapters1(index, &adapter); - if (result == DXGI_ERROR_NOT_FOUND) { - break; - } - Check(result, "dxgi-enumerate-adapter"); - - Adapter current; - current.index = index; - current.value = adapter; - Check(adapter->GetDesc1(¤t.description), "dxgi-adapter-description"); - if ((current.description.Flags & DXGI_ADAPTER_FLAG_SOFTWARE) == 0) { - adapters.push_back(std::move(current)); - } - } - return adapters; -} - void PrintAdapters(const std::vector& adapters) { for (const Adapter& adapter : adapters) { std::cout << "adapter_index=" << adapter.index @@ -281,402 +139,6 @@ void PrintAdapters(const std::vector& adapters) { } } -const Adapter& SelectAdapter(const std::vector& adapters, UINT index) { - auto found = std::find_if(adapters.begin(), adapters.end(), - [index](const Adapter& adapter) { - return adapter.index == index; - }); - if (found == adapters.end()) { - Fail("adapter-select", "selected DXGI adapter index is unavailable"); - } - return *found; -} - -struct DeviceContext final { - ComPtr device; - ComPtr context; - ComPtr manager; - D3D_FEATURE_LEVEL feature_level = D3D_FEATURE_LEVEL_9_1; -}; - -DeviceContext CreateDevice(const Adapter& adapter) { - DeviceContext result; - constexpr D3D_FEATURE_LEVEL levels[] = { - D3D_FEATURE_LEVEL_12_1, D3D_FEATURE_LEVEL_12_0, - D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0, - }; - Check(D3D11CreateDevice(adapter.value.Get(), D3D_DRIVER_TYPE_UNKNOWN, nullptr, - D3D11_CREATE_DEVICE_VIDEO_SUPPORT | - D3D11_CREATE_DEVICE_BGRA_SUPPORT, - levels, static_cast(std::size(levels)), - D3D11_SDK_VERSION, &result.device, - &result.feature_level, &result.context), - "d3d11-create-device"); - - ComPtr multithread; - Check(result.device.As(&multithread), "d3d11-multithread-interface"); - multithread->SetMultithreadProtected(TRUE); - if (!multithread->GetMultithreadProtected()) { - Fail("d3d11-multithread-protection", - "D3D11 device did not enable multithread protection"); - } - - UINT reset_token = 0; - Check(MFCreateDXGIDeviceManager(&reset_token, &result.manager), - "dxgi-device-manager-create"); - Check(result.manager->ResetDevice(result.device.Get(), reset_token), - "dxgi-device-manager-reset"); - return result; -} - -class ActivationList final { - public: - ActivationList() = default; - ActivationList(const ActivationList&) = delete; - ActivationList& operator=(const ActivationList&) = delete; - ActivationList(ActivationList&& other) noexcept - : values(std::exchange(other.values, nullptr)), - count(std::exchange(other.count, 0)) {} - - ~ActivationList() { - for (UINT32 index = 0; index < count; ++index) { - values[index]->Release(); - } - CoTaskMemFree(values); - } - - IMFActivate** values = nullptr; - UINT32 count = 0; -}; - -std::wstring GetOptionalString(IMFAttributes* attributes, const GUID& key) { - wchar_t* raw = nullptr; - UINT32 length = 0; - HRESULT result = attributes->GetAllocatedString(key, &raw, &length); - if (result == MF_E_ATTRIBUTENOTFOUND) { - return L""; - } - Check(result, "mft-activation-string"); - std::wstring value(raw, length); - CoTaskMemFree(raw); - return value; -} - -ActivationList EnumerateHardwareEncoders(const Adapter& adapter, - bool require_candidate = true) { - ComPtr attributes; - Check(MFCreateAttributes(&attributes, 1), "mft-enumeration-attributes"); - Check(attributes->SetBlob(MFT_ENUM_ADAPTER_LUID, - reinterpret_cast( - &adapter.description.AdapterLuid), - sizeof(LUID)), - "mft-enumeration-adapter-luid"); - - MFT_REGISTER_TYPE_INFO input = {MFMediaType_Video, MFVideoFormat_NV12}; - MFT_REGISTER_TYPE_INFO output = {MFMediaType_Video, MFVideoFormat_H264}; - ActivationList list; - Check(MFTEnum2(MFT_CATEGORY_VIDEO_ENCODER, - MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_SORTANDFILTER, - &input, &output, attributes.Get(), &list.values, &list.count), - "mft-hardware-enumeration"); - if (require_candidate && list.count == 0) { - Fail("mft-hardware-enumeration", - "selected adapter exposed no hardware NV12-to-H264 MFT"); - } - return list; -} - -struct SelectedTransform final { - std::string name; - std::string clsid; - ComPtr transform; - ComPtr codec; - ComPtr events; -}; - -SelectedTransform ActivateTransform(const ActivationList& list, UINT index, - IMFDXGIDeviceManager* manager) { - if (index >= list.count) { - Fail("mft-select", "selected hardware MFT index is unavailable"); - } - - IMFActivate* activation = list.values[index]; - std::wstring friendly = GetOptionalString(activation, MFT_FRIENDLY_NAME_Attribute); - if (friendly.empty()) { - Fail("mft-identity", "hardware MFT has no friendly-name attribute"); - } - - GUID clsid = {}; - Check(activation->GetGUID(MFT_TRANSFORM_CLSID_Attribute, &clsid), - "mft-clsid"); - - PROPVARIANT hardware_url; - PropVariantInit(&hardware_url); - HRESULT hardware_result = activation->GetItem( - MFT_ENUM_HARDWARE_URL_Attribute, &hardware_url); - PropVariantClear(&hardware_url); - if (FAILED(hardware_result)) { - Fail("mft-hardware-identity", - "hardware-only activation omitted its hardware URL attribute"); - } - - SelectedTransform selected; - selected.name = NarrowAscii(friendly); - selected.clsid = GuidString(clsid); - Check(activation->ActivateObject(IID_PPV_ARGS(&selected.transform)), - "mft-activate"); - - ComPtr transform_attributes; - Check(selected.transform->GetAttributes(&transform_attributes), - "mft-global-attributes"); - UINT32 is_async = 0; - Check(transform_attributes->GetUINT32(MF_TRANSFORM_ASYNC, &is_async), - "mft-async-readback"); - if (is_async != TRUE) { - Fail("mft-async-readback", "hardware MFT is not asynchronous"); - } - Check(transform_attributes->SetUINT32(MF_TRANSFORM_ASYNC_UNLOCK, TRUE), - "mft-async-unlock"); - - UINT32 d3d11_aware = 0; - Check(transform_attributes->GetUINT32(MF_SA_D3D11_AWARE, &d3d11_aware), - "mft-d3d11-aware-readback"); - if (d3d11_aware != TRUE) { - Fail("mft-d3d11-aware-readback", "hardware MFT is not D3D11-aware"); - } - - Check(selected.transform->ProcessMessage( - MFT_MESSAGE_SET_D3D_MANAGER, - reinterpret_cast(manager)), - "mft-set-d3d-manager"); - Check(selected.transform.As(&selected.codec), "mft-codec-api"); - Check(selected.transform.As(&selected.events), "mft-event-generator"); - return selected; -} - -void RequireProperty(ICodecAPI* codec, const GUID& key, - const std::string& name) { - HRESULT supported = codec->IsSupported(&key); - if (supported != S_OK) { - if (FAILED(supported) && supported != E_NOTIMPL) { - Check(supported, name + "-supported-query"); - } - Fail(name + "-unsupported", "required codec property is unsupported"); - } - HRESULT modifiable = codec->IsModifiable(&key); - if (modifiable == S_FALSE) { - Fail(name + "-readonly", "required codec property is read-only"); - } - // Some hardware MFTs do not implement this advisory query. SetValue=S_OK - // followed by exact GetValue readback remains the authoritative gate. - if (modifiable != S_OK && modifiable != E_NOTIMPL) { - Check(modifiable, name + "-modifiable-query"); - } -} - -UINT32 ReadU32(ICodecAPI* codec, const GUID& key, const std::string& name) { - VARIANT value; - VariantInit(&value); - Check(codec->GetValue(&key, &value), name + "-readback"); - if (value.vt != VT_UI4) { - VariantClear(&value); - Fail(name + "-readback-type", "codec property did not return VT_UI4"); - } - UINT32 result = value.ulVal; - VariantClear(&value); - return result; -} - -bool ReadBool(ICodecAPI* codec, const GUID& key, const std::string& name) { - VARIANT value; - VariantInit(&value); - Check(codec->GetValue(&key, &value), name + "-readback"); - if (value.vt != VT_BOOL) { - VariantClear(&value); - Fail(name + "-readback-type", "codec property did not return VT_BOOL"); - } - bool result = value.boolVal != VARIANT_FALSE; - VariantClear(&value); - return result; -} - -void SetU32(ICodecAPI* codec, const GUID& key, UINT32 expected, - const std::string& name) { - RequireProperty(codec, key, name); - VARIANT value; - VariantInit(&value); - value.vt = VT_UI4; - value.ulVal = expected; - HRESULT set_result = codec->SetValue(&key, &value); - VariantClear(&value); - if (set_result != S_OK) { - if (FAILED(set_result)) { - Check(set_result, name + "-set"); - } - Fail(name + "-readonly", "codec property rejected the requested value"); - } - if (ReadU32(codec, key, name) != expected) { - Fail(name + "-weakened", "codec property readback changed the requested value"); - } -} - -void SetBool(ICodecAPI* codec, const GUID& key, bool expected, - const std::string& name) { - RequireProperty(codec, key, name); - VARIANT value; - VariantInit(&value); - value.vt = VT_BOOL; - value.boolVal = expected ? VARIANT_TRUE : VARIANT_FALSE; - HRESULT set_result = codec->SetValue(&key, &value); - VariantClear(&value); - if (set_result != S_OK) { - if (FAILED(set_result)) { - Check(set_result, name + "-set"); - } - Fail(name + "-readonly", "codec property rejected the requested value"); - } - if (ReadBool(codec, key, name) != expected) { - Fail(name + "-weakened", "codec property readback changed the requested value"); - } -} - -void SetOptionalU32(ICodecAPI* codec, const GUID& key, UINT32 value) { - if (codec->IsSupported(&key) != S_OK) return; - VARIANT requested; - VariantInit(&requested); - requested.vt = VT_UI4; - requested.ulVal = value; - (void)codec->SetValue(&key, &requested); - VariantClear(&requested); -} - -void ConfigureCodec( - ICodecAPI* codec, - const VideoProfile& profile = kDefaultVideoProfile) { - SetU32(codec, CODECAPI_AVEncCommonRateControlMode, - eAVEncCommonRateControlMode_CBR, "codec-cbr"); - SetBool(codec, CODECAPI_AVLowLatencyMode, true, "codec-low-latency"); - SetU32(codec, CODECAPI_AVEncCommonMeanBitRate, profile.bit_rate, - "codec-mean-bitrate"); - SetU32(codec, CODECAPI_AVEncCommonBufferSize, profile.vbv_bytes(), - "codec-vbv-bytes"); - SetU32(codec, CODECAPI_AVEncMPVGOPSize, profile.gop_frames(), "codec-gop"); - SetOptionalU32(codec, CODECAPI_AVEncCommonQualityVsSpeed, - profile.quality_vs_speed()); - RequireProperty(codec, CODECAPI_AVEncVideoForceKeyFrame, - "codec-force-keyframe"); -} - -ComPtr CreateOutputType( - const VideoProfile& profile = kDefaultVideoProfile) { - ComPtr type; - Check(MFCreateMediaType(&type), "output-type-create"); - Check(type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video), - "output-type-major"); - Check(type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264), - "output-type-subtype"); - Check(type->SetUINT32(MF_MT_AVG_BITRATE, profile.bit_rate), - "output-type-bitrate"); - Check(MFSetAttributeSize(type.Get(), MF_MT_FRAME_SIZE, profile.width, - profile.height), - "output-type-size"); - Check(MFSetAttributeRatio(type.Get(), MF_MT_FRAME_RATE, profile.frame_rate, 1), - "output-type-framerate"); - Check(MFSetAttributeRatio(type.Get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1), - "output-type-pixel-aspect"); - Check(type->SetUINT32(MF_MT_INTERLACE_MODE, - MFVideoInterlace_Progressive), - "output-type-interlace"); - Check(type->SetUINT32(MF_MT_MPEG2_PROFILE, eAVEncH264VProfile_Base), - "output-type-profile"); - Check(type->SetUINT32(MF_MT_MPEG2_LEVEL, profile.h264_level()), - "output-type-level"); - return type; -} - -ComPtr CreateInputType( - const VideoProfile& profile = kDefaultVideoProfile) { - ComPtr type; - Check(MFCreateMediaType(&type), "input-type-create"); - Check(type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video), - "input-type-major"); - Check(type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12), - "input-type-subtype"); - Check(MFSetAttributeSize(type.Get(), MF_MT_FRAME_SIZE, profile.width, - profile.height), - "input-type-size"); - Check(MFSetAttributeRatio(type.Get(), MF_MT_FRAME_RATE, profile.frame_rate, 1), - "input-type-framerate"); - Check(MFSetAttributeRatio(type.Get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1), - "input-type-pixel-aspect"); - Check(type->SetUINT32(MF_MT_INTERLACE_MODE, - MFVideoInterlace_Progressive), - "input-type-interlace"); - Check(type->SetUINT32(MF_MT_DEFAULT_STRIDE, profile.width), - "input-type-stride"); - return type; -} - -void RequireTypeU32(IMFAttributes* type, const GUID& key, UINT32 expected, - const std::string& stage) { - UINT32 actual = 0; - Check(type->GetUINT32(key, &actual), stage + "-readback"); - if (actual != expected) { - Fail(stage + "-weakened", "media type changed a required value"); - } -} - -void ValidateMediaTypes( - IMFTransform* transform, - const VideoProfile& profile = kDefaultVideoProfile) { - ComPtr output; - Check(transform->GetOutputCurrentType(0, &output), - "output-type-current"); - GUID subtype = {}; - Check(output->GetGUID(MF_MT_SUBTYPE, &subtype), "output-type-subtype-readback"); - if (subtype != MFVideoFormat_H264) { - Fail("output-type-subtype-weakened", "output is not H264"); - } - RequireTypeU32(output.Get(), MF_MT_AVG_BITRATE, profile.bit_rate, - "output-type-bitrate"); - RequireTypeU32(output.Get(), MF_MT_MPEG2_PROFILE, - eAVEncH264VProfile_Base, "output-type-profile"); - RequireTypeU32(output.Get(), MF_MT_MPEG2_LEVEL, profile.h264_level(), - "output-type-level"); - - UINT32 width = 0; - UINT32 height = 0; - Check(MFGetAttributeSize(output.Get(), MF_MT_FRAME_SIZE, &width, &height), - "output-type-size-readback"); - if (width != profile.width || height != profile.height) { - Fail("output-type-size-weakened", "output dimensions changed"); - } - UINT32 numerator = 0; - UINT32 denominator = 0; - Check(MFGetAttributeRatio(output.Get(), MF_MT_FRAME_RATE, &numerator, - &denominator), - "output-type-framerate-readback"); - if (numerator != profile.frame_rate || denominator != 1) { - Fail("output-type-framerate-weakened", "output frame rate changed"); - } -} - -void ValidateCodecReadback( - ICodecAPI* codec, - const VideoProfile& profile = kDefaultVideoProfile) { - if (ReadU32(codec, CODECAPI_AVEncCommonRateControlMode, "codec-cbr") != - eAVEncCommonRateControlMode_CBR || - !ReadBool(codec, CODECAPI_AVLowLatencyMode, "codec-low-latency") || - ReadU32(codec, CODECAPI_AVEncCommonMeanBitRate, - "codec-mean-bitrate") != profile.bit_rate || - ReadU32(codec, CODECAPI_AVEncCommonBufferSize, "codec-vbv-bytes") != - profile.vbv_bytes() || - ReadU32(codec, CODECAPI_AVEncMPVGOPSize, "codec-gop") != - profile.gop_frames()) { - Fail("codec-final-readback", "one or more codec properties were weakened"); - } -} - std::vector SyntheticNv12( UINT32 frame_index, const VideoProfile& profile = kDefaultVideoProfile) { const auto width = profile.width; @@ -744,92 +206,6 @@ ComPtr CreateInputSample(ID3D11Device* device, UINT32 frame_index) { } #endif -struct NalSummary final { - bool annex_b = false; - bool sps = false; - bool pps = false; - bool idr = false; - std::optional profile_level_id; -}; - -std::optional> FindStartCode( - const std::vector& bytes, size_t from) { - for (size_t index = from; index + 3 <= bytes.size(); ++index) { - if (bytes[index] != 0 || bytes[index + 1] != 0) { - continue; - } - if (bytes[index + 2] == 1) { - return std::pair{index, 3}; - } - if (index + 4 <= bytes.size() && bytes[index + 2] == 0 && - bytes[index + 3] == 1) { - return std::pair{index, 4}; - } - } - return std::nullopt; -} - -NalSummary InspectAnnexB(const std::vector& bytes) { - NalSummary summary; - std::optional> current = FindStartCode(bytes, 0); - if (!current || current->first != 0) { - return summary; - } - summary.annex_b = true; - while (current) { - size_t nal_start = current->first + current->second; - std::optional> next = FindStartCode(bytes, nal_start); - size_t nal_end = next ? next->first : bytes.size(); - if (nal_start < nal_end) { - UINT8 nal_type = bytes[nal_start] & 0x1f; - summary.sps = summary.sps || nal_type == 7; - summary.pps = summary.pps || nal_type == 8; - summary.idr = summary.idr || nal_type == 5; - if (nal_type == 7 && nal_end - nal_start >= 4) { - std::string value = ProfileLevelId(bytes[nal_start + 1], - bytes[nal_start + 2], - bytes[nal_start + 3]); - if (summary.profile_level_id && *summary.profile_level_id != value) { - Fail("bitstream-sps-consistency", - "one access unit contained conflicting SPS profile bytes"); - } - summary.profile_level_id = value; - } - } - current = next; - } - return summary; -} - -std::vector ReadSample(IMFSample* sample) { - DWORD total_length = 0; - Check(sample->GetTotalLength(&total_length), "output-total-length"); - if (total_length == 0 || total_length > kMaxEncodedSampleBytes) { - Fail("output-buffer-bounds", "encoded output size is empty or unbounded"); - } - - ComPtr buffer; - Check(sample->ConvertToContiguousBuffer(&buffer), "output-contiguous-buffer"); - DWORD length = 0; - Check(buffer->GetCurrentLength(&length), "output-buffer-length"); - if (length != total_length) { - Fail("output-buffer-length", "contiguous output length changed unexpectedly"); - } - std::vector bytes(length); - UINT8* raw = nullptr; - DWORD maximum = 0; - DWORD current = 0; - Check(buffer->Lock(&raw, &maximum, ¤t), "output-buffer-lock"); - if (raw == nullptr || current != length || current > kMaxEncodedSampleBytes) { - buffer->Unlock(); - Fail("output-buffer-lock-bounds", - "locked output length is inconsistent or unbounded"); - } - std::copy(raw, raw + current, bytes.begin()); - Check(buffer->Unlock(), "output-buffer-unlock"); - return bytes; -} - #ifdef SCREENER_H264_FIXTURE ULONGLONG ProcessCpu100ns() { FILETIME created = {}; @@ -991,82 +367,6 @@ struct RunEvidence final { }; #endif -void ForceKeyFrame(ICodecAPI* codec) { - VARIANT value; - VariantInit(&value); - value.vt = VT_UI4; - value.ulVal = 1; - HRESULT set_result = - codec->SetValue(&CODECAPI_AVEncVideoForceKeyFrame, &value); - VariantClear(&value); - if (set_result != S_OK) { - if (FAILED(set_result)) { - Check(set_result, "codec-force-keyframe-set"); - } - Fail("codec-force-keyframe-readonly", - "force-key-frame command was not accepted"); - } -} - -ComPtr CreateCallerOutputSample(const MFT_OUTPUT_STREAM_INFO& info) { - if (info.cbSize > kMaxEncodedSampleBytes) { - Fail("output-buffer-contract", - "hardware MFT requested an output buffer larger than four MiB"); - } - ComPtr sample; - Check(MFCreateSample(&sample), "output-sample-create"); - ComPtr buffer; - Check(MFCreateMemoryBuffer(kMaxEncodedSampleBytes, &buffer), - "output-memory-buffer"); - Check(sample->AddBuffer(buffer.Get()), "output-sample-buffer"); - return sample; -} - -ComPtr PullOutput(IMFTransform* transform, - const MFT_OUTPUT_STREAM_INFO& info) { - bool transform_provides = - (info.dwFlags & (MFT_OUTPUT_STREAM_PROVIDES_SAMPLES | - MFT_OUTPUT_STREAM_CAN_PROVIDE_SAMPLES)) != 0; - ComPtr caller_sample; - if (!transform_provides) { - caller_sample = CreateCallerOutputSample(info); - } - - MFT_OUTPUT_DATA_BUFFER output = {}; - output.dwStreamID = 0; - output.pSample = caller_sample.Get(); - DWORD status = 0; - HRESULT result = transform->ProcessOutput(0, 1, &output, &status); - - ComPtr events; - if (output.pEvents != nullptr) { - events.Attach(output.pEvents); - } - Check(result, "mft-process-output"); - if ((output.dwStatus & MFT_OUTPUT_DATA_BUFFER_FORMAT_CHANGE) != 0 || - (status & MFT_PROCESS_OUTPUT_STATUS_NEW_STREAMS) != 0) { - Fail("mft-output-format-change", - "hardware MFT changed format during the fixed run"); - } - - if (caller_sample) { - if (output.pSample != caller_sample.Get()) { - if (output.pSample != nullptr) { - output.pSample->Release(); - } - Fail("mft-output-allocation", - "hardware MFT replaced the caller-owned output sample"); - } - return caller_sample; - } - if (output.pSample == nullptr) { - Fail("mft-output-sample", "hardware MFT produced no output sample"); - } - ComPtr produced; - produced.Attach(output.pSample); - return produced; -} - #ifdef SCREENER_H264_FIXTURE bool IsRecoveryFrame(UINT32 frame) { return frame % kGopFrames == 0; @@ -1336,14 +636,9 @@ UINT ParseIndex(const wchar_t* value, const std::string& stage) { } #ifndef SCREENER_H264_FIXTURE -constexpr DWORD kMaxProductAccessUnitBytes = 1 * 1024 * 1024; -constexpr DWORD kMaxStatusBytes = 4 * 1024; -enum class OutputKind : UINT8 { - pcm = 1, - h264 = 2, - status = 3, - vp8 = 4, -}; +using screener::capture::kMaxProductAccessUnitBytes; +using screener::capture::OutputKind; +using screener::capture::ProtocolWriter; class UniqueHandle final { public: @@ -1362,301 +657,6 @@ class UniqueHandle final { HANDLE value_ = nullptr; }; -void PutUint32BE(BYTE* output, UINT32 value) { - for (int index = 3; index >= 0; --index) { - output[index] = static_cast(value & 0xff); - value >>= 8; - } -} - -void PutUint64BE(BYTE* output, UINT64 value) { - for (int index = 7; index >= 0; --index) { - output[index] = static_cast(value & 0xff); - value >>= 8; - } -} - -class ProtocolWriter final { - public: - HRESULT Write(OutputKind kind, UINT8 flags, UINT64 timestamp100ns, - UINT64 duration100ns, const BYTE* data, DWORD size) { - DWORD maximum = kind == OutputKind::status ? kMaxStatusBytes - : kMaxProductAccessUnitBytes; - if (data == nullptr || size == 0 || size > maximum || - (kind != OutputKind::status && duration100ns == 0)) { - return HRESULT_FROM_WIN32(ERROR_INVALID_DATA); - } - std::array header{}; - header[0] = 'S'; - header[1] = 'M'; - header[2] = 'E'; - header[3] = 'D'; - header[4] = 1; - header[5] = static_cast(kind); - header[6] = flags; - PutUint64BE(header.data() + 8, timestamp100ns); - PutUint64BE(header.data() + 16, duration100ns); - PutUint32BE(header.data() + 24, size); - - std::lock_guard lock(mutex_); - HRESULT result = WriteAll(header.data(), static_cast(header.size())); - return SUCCEEDED(result) ? WriteAll(data, size) : result; - } - - HRESULT WriteStatus(const std::string& value) { - return Write(OutputKind::status, 0, 0, 0, - reinterpret_cast(value.data()), - static_cast(value.size())); - } - - private: - HRESULT WriteAll(const BYTE* data, DWORD size) { - HANDLE output = GetStdHandle(STD_OUTPUT_HANDLE); - while (size > 0) { - DWORD written = 0; - if (!WriteFile(output, data, size, &written, nullptr)) { - return HRESULT_FROM_WIN32(GetLastError()); - } - if (written == 0) return HRESULT_FROM_WIN32(ERROR_WRITE_FAULT); - data += written; - size -= written; - } - return S_OK; - } - - std::mutex mutex_; -}; - -ComPtr CreateSurfaceSample(ID3D11Texture2D* texture, - LONGLONG timestamp100ns, - LONGLONG duration100ns) { - ComPtr buffer; - Check(MFCreateDXGISurfaceBuffer(IID_ID3D11Texture2D, texture, 0, FALSE, - &buffer), - "input-dxgi-buffer"); - ComPtr sample; - Check(MFCreateVideoSampleFromSurface(nullptr, &sample), - "input-video-sample"); - Check(sample->AddBuffer(buffer.Get()), "input-sample-buffer"); - Check(sample->SetSampleTime(timestamp100ns), "input-sample-time"); - Check(sample->SetSampleDuration(duration100ns), "input-sample-duration"); - return sample; -} - -struct EncodedAccessUnit final { - UINT64 timestamp100ns = 0; - bool key_frame = false; - std::vector bytes; -}; - -using EncoderClock = std::chrono::steady_clock; - -void RequireEncoderTime(EncoderClock::time_point deadline) { - if (EncoderClock::now() >= deadline) { - Fail("codec-probe-timeout", "native codec selection exceeded its time budget"); - } -} - -class VideoEncoder { - public: - VideoEncoder(OutputKind kind, std::string name, std::string identity) - : kind(kind), name(std::move(name)), identity(std::move(identity)) {} - virtual ~VideoEncoder() = default; - virtual EncodedAccessUnit Encode(ID3D11Texture2D* texture, - UINT64 timestamp100ns, - bool force_key_frame, - EncoderClock::time_point deadline = - EncoderClock::time_point::max()) = 0; - const OutputKind kind; - const std::string name; - const std::string identity; -}; - -class LiveEncoder final : public VideoEncoder { - public: - explicit LiveEncoder( - SelectedTransform selected, - VideoProfile profile = kDefaultVideoProfile) - : VideoEncoder(OutputKind::h264, selected.name, selected.clsid), - selected_(std::move(selected)), profile_(profile) { - ConfigureCodec(selected_.codec.Get(), profile_); - ComPtr output_type = CreateOutputType(profile_); - Check(selected_.transform->SetOutputType(0, output_type.Get(), 0), - "mft-set-output-type"); - ComPtr input_type = CreateInputType(profile_); - Check(selected_.transform->SetInputType(0, input_type.Get(), 0), - "mft-set-input-type"); - ValidateMediaTypes(selected_.transform.Get(), profile_); - ValidateCodecReadback(selected_.codec.Get(), profile_); - Check(selected_.transform->GetOutputStreamInfo(0, &output_info_), - "mft-output-stream-info"); - Check(selected_.transform->ProcessMessage( - MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0), - "mft-begin-streaming"); - Check(selected_.transform->ProcessMessage( - MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0), - "mft-start-stream"); - started_ = true; - } - - LiveEncoder(const LiveEncoder&) = delete; - LiveEncoder& operator=(const LiveEncoder&) = delete; - - ~LiveEncoder() override { - if (started_) { - selected_.transform->ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0); - selected_.transform->ProcessMessage(MFT_MESSAGE_NOTIFY_END_STREAMING, 0); - } - } - - EncodedAccessUnit Encode(ID3D11Texture2D* texture, UINT64 timestamp100ns, - bool force_key_frame, - EncoderClock::time_point probe_deadline) override { - WaitForInput(probe_deadline); - if (force_key_frame) ForceKeyFrame(selected_.codec.Get()); - ComPtr sample = CreateSurfaceSample( - texture, static_cast(timestamp100ns), - profile_.frame_duration_100ns()); - Check(selected_.transform->ProcessInput(0, sample.Get(), 0), - "mft-process-input"); - --input_requests_; - - const auto deadline = std::min(probe_deadline, - EncoderClock::now() + std::chrono::seconds(2)); - while (std::chrono::steady_clock::now() < deadline) { - MediaEventType type = NextEvent(deadline); - if (type == METransformNeedInput) { - ++input_requests_; - continue; - } - if (type != METransformHaveOutput) continue; - - ComPtr output = PullOutput(selected_.transform.Get(), - output_info_); - LONGLONG output_time = -1; - Check(output->GetSampleTime(&output_time), "output-sample-time"); - if (output_time != static_cast(timestamp100ns)) { - Fail("output-order", "hardware MFT changed or reordered a live timestamp"); - } - std::vector bytes = ReadSample(output.Get()); - NalSummary nal = InspectAnnexB(bytes); - if (!nal.annex_b) { - Fail("bitstream-annexb", "live output is not Annex-B H264"); - } - if (nal.profile_level_id) { - if (*nal.profile_level_id != profile_.profile_level_id()) { - Fail("bitstream-profile", "hardware MFT changed the requested H.264 profile level"); - } - if (profile_level_id_ && *profile_level_id_ != *nal.profile_level_id) { - Fail("bitstream-profile-change", "hardware MFT changed SPS profile"); - } - profile_level_id_ = nal.profile_level_id; - } - if (force_key_frame && !(nal.sps && nal.pps && nal.idr)) { - Fail("bitstream-recovery-unit", - "requested live recovery unit lacks SPS, PPS, or IDR"); - } - if (!profile_level_id_) { - Fail("bitstream-sps", "first live access unit did not carry an SPS"); - } - RequireEncoderTime(probe_deadline); - return EncodedAccessUnit{timestamp100ns, nal.idr, std::move(bytes)}; - } - Fail("mft-output-timeout", "hardware MFT did not produce live output in time"); - } - - private: - MediaEventType NextEvent(std::chrono::steady_clock::time_point deadline) { - while (std::chrono::steady_clock::now() < deadline) { - ComPtr event; - HRESULT result = selected_.events->GetEvent(MF_EVENT_FLAG_NO_WAIT, - &event); - if (result == MF_E_NO_EVENTS_AVAILABLE) { - std::this_thread::sleep_for(std::chrono::milliseconds(1)); - continue; - } - Check(result, "mft-get-event"); - HRESULT status = S_OK; - Check(event->GetStatus(&status), "mft-event-status-read"); - Check(status, "mft-event-status"); - MediaEventType type = MEUnknown; - Check(event->GetType(&type), "mft-event-type"); - if (type == MEError) Fail("mft-error-event", "hardware MFT emitted MEError"); - return type; - } - Fail("mft-event-timeout", "hardware MFT did not request input in time"); - } - - void WaitForInput(EncoderClock::time_point probe_deadline) { - const auto deadline = std::min(probe_deadline, - EncoderClock::now() + std::chrono::seconds(2)); - while (input_requests_ == 0) { - MediaEventType type = NextEvent(deadline); - if (type == METransformNeedInput) { - ++input_requests_; - } else if (type == METransformHaveOutput) { - Fail("mft-unexpected-output", "hardware MFT produced output without live input"); - } - } - } - - SelectedTransform selected_; - VideoProfile profile_; - MFT_OUTPUT_STREAM_INFO output_info_ = {}; - UINT32 input_requests_ = 0; - std::optional profile_level_id_; - bool started_ = false; -}; - -class SoftwareVp8Encoder final : public VideoEncoder { - public: - SoftwareVp8Encoder(ID3D11Device* device, const VideoProfile& profile) - : VideoEncoder(OutputKind::vp8, "libvpx VP8", - std::string("libvpx/") + screener::capture::Vp8Encoder::Version()), - encoder_(profile.width, profile.height, profile.frame_rate, profile.bit_rate) { - device->GetImmediateContext(&context_); - D3D11_TEXTURE2D_DESC description{}; - description.Width = profile.width; - description.Height = profile.height; - description.MipLevels = 1; - description.ArraySize = 1; - description.Format = DXGI_FORMAT_NV12; - description.SampleDesc.Count = 1; - description.Usage = D3D11_USAGE_STAGING; - description.CPUAccessFlags = D3D11_CPU_ACCESS_READ; - Check(device->CreateTexture2D(&description, nullptr, &staging_), - "vp8-readback-texture"); - } - - EncodedAccessUnit Encode(ID3D11Texture2D* texture, UINT64 timestamp100ns, - bool force_key_frame, - EncoderClock::time_point deadline) override { - RequireEncoderTime(deadline); - context_->CopyResource(staging_.Get(), texture); - D3D11_MAPPED_SUBRESOURCE mapped{}; - Check(context_->Map(staging_.Get(), 0, D3D11_MAP_READ, 0, &mapped), - "vp8-readback-map"); - screener::capture::Vp8Frame frame; - try { - RequireEncoderTime(deadline); - frame = encoder_.Encode(static_cast(mapped.pData), - mapped.RowPitch, timestamp100ns, - force_key_frame); - } catch (...) { - context_->Unmap(staging_.Get(), 0); - throw; - } - context_->Unmap(staging_.Get(), 0); - RequireEncoderTime(deadline); - return {timestamp100ns, frame.key_frame, std::move(frame.bytes)}; - } - - private: - screener::capture::Vp8Encoder encoder_; - ComPtr context_; - ComPtr staging_; -}; - class FrameConverter final { public: explicit FrameConverter( @@ -1839,41 +839,50 @@ std::string JSONString(const std::string& value) { return output; } -struct ControlSignal final { - bool key_frame = false; - bool stop = false; +class InputReader final { + public: + std::vector Read(bool blocking = false) { + HANDLE input = GetStdHandle(STD_INPUT_HANDLE); + DWORD available = 0; + if (input == INVALID_HANDLE_VALUE || input == nullptr) return {}; + if (!blocking && !PeekNamedPipe(input, nullptr, 0, nullptr, &available, nullptr)) { + if (GetLastError() == ERROR_BROKEN_PIPE) return End(); + Check(HRESULT_FROM_WIN32(GetLastError()), "capture-control-peek"); + } + if (!blocking && available == 0) return {}; + std::array bytes{}; + DWORD read = 0; + if (!ReadFile(input, bytes.data(), + blocking ? static_cast(bytes.size()) : std::min(available, static_cast(bytes.size())), + &read, nullptr)) { + if (GetLastError() == ERROR_BROKEN_PIPE) return End(); + Check(HRESULT_FROM_WIN32(GetLastError()), "capture-control-read"); + } + return envelopes_.Feed(std::string_view(bytes.data(), read)); + } + + private: + std::vector End() { + if (!envelopes_.Empty()) Fail("capture-input-eof", "native input ended within an envelope"); + screener::capture::InputEnvelope stop; + stop.kind = 7; + stop.data = {'Q'}; + return {std::move(stop)}; + } + screener::capture::InputEnvelopes envelopes_; }; -ControlSignal ConsumeControlSignal() { - HANDLE input = GetStdHandle(STD_INPUT_HANDLE); - DWORD available = 0; - if (input == INVALID_HANDLE_VALUE || input == nullptr) { - return {}; +class ControlReader final { + public: + std::vector Read() { + std::vector controls; + for (const auto& envelope : input_.Read()) controls.push_back(envelope.Control()); + return controls; } - if (!PeekNamedPipe(input, nullptr, 0, nullptr, &available, nullptr)) { - DWORD error = GetLastError(); - return {.stop = error == ERROR_BROKEN_PIPE}; - } - if (available == 0) { - return {}; - } - std::array bytes{}; - DWORD read = 0; - if (!ReadFile(input, bytes.data(), - std::min(available, static_cast(bytes.size())), - &read, nullptr)) { - return {.stop = GetLastError() == ERROR_BROKEN_PIPE}; - } - ControlSignal signal; - for (auto current = bytes.begin(); current != bytes.begin() + read; - ++current) { - signal.key_frame = signal.key_frame || *current == static_cast('K'); - signal.stop = signal.stop || *current == static_cast('Q') || - *current == static_cast('\n') || - *current == static_cast('\r'); - } - return signal; -} + + private: + InputReader input_; +}; UINT64 ParseUint64(const wchar_t* value, const std::string& stage) { try { @@ -1909,7 +918,7 @@ screener::capture::TargetKind ParseTargetKind(const wchar_t* value) { } struct ProductArguments final { - enum class Mode { list, probe, preview, audio, video } mode = Mode::list; + enum class Mode { list, probe, preview, audio, video, encoded } mode = Mode::list; screener::capture::TargetKind target_kind = screener::capture::TargetKind::window; DWORD pid = 0; @@ -1919,6 +928,7 @@ struct ProductArguments final { UINT mft_index = 0; std::string codec = "auto"; VideoProfile profile; + std::vector outputs; }; DegradationPreference ParseDegradationPreference(const wchar_t* value) { @@ -1945,6 +955,28 @@ void ValidateVideoProfile(const VideoProfile& profile) { } } +void ParseOutputProfiles(ProductArguments& arguments, int first, int count, wchar_t** values) { + for (int index = first; index < count; index += 5) { + if (std::wstring(values[index]) != L"--output") Fail("argument-output", "output profile is missing"); + VideoProfile output = arguments.profile; + output.width = ParseIndex(values[index + 1], "argument-output-width"); + output.height = ParseIndex(values[index + 2], "argument-output-height"); + output.frame_rate = ParseIndex(values[index + 3], "argument-output-fps"); + output.bit_rate = ParseIndex(values[index + 4], "argument-output-bitrate"); + if (output.width < 2 || output.height < 2 || (output.width & 1) || (output.height & 1) || + output.width > 2560 || output.height > 1440 || output.frame_rate == 0 || + output.frame_rate > 60 || output.bit_rate < 1'000 || output.bit_rate > 12'000'000) { + Fail("argument-output", "output profile exceeds codec bounds"); + } + if (arguments.mode == ProductArguments::Mode::video && + (output.width > arguments.profile.width || output.height > arguments.profile.height || + output.frame_rate > arguments.profile.frame_rate || output.bit_rate > arguments.profile.bit_rate)) { + Fail("argument-output", "output profile exceeds source bounds"); + } + arguments.outputs.push_back(output); + } +} + ProductArguments ParseProductArguments(int count, wchar_t** values) { ProductArguments arguments; if (count == 2 && std::wstring(values[1]) == L"--list") return arguments; @@ -1968,7 +1000,30 @@ ProductArguments ParseProductArguments(int count, wchar_t** values) { if (count == 5 && std::wstring(values[1]) == L"--capture-audio") { arguments.mode = ProductArguments::Mode::audio; arguments.target_kind = ParseTargetKind(values[2]); - } else if (count == 23 && std::wstring(values[1]) == L"--capture-video" && + } else if (count >= 16 && count <= 11 + 5 * screener::capture::kMaxOutputs && (count - 11) % 5 == 0 && + std::wstring(values[1]) == L"--encoded-video" && + std::wstring(values[2]) == L"--codec" && + std::wstring(values[4]) == L"--adapter-index" && + std::wstring(values[6]) == L"--mft-index" && + std::wstring(values[8]) == L"--preference" && + std::wstring(values[10]) == L"--protocol-v7") { + arguments.mode = ProductArguments::Mode::encoded; + arguments.codec = NarrowAscii(values[3]); + if (arguments.codec != "h264" && arguments.codec != "vp8") Fail("argument-codec", "encoded input codec is unsupported"); + arguments.adapter_index = ParseIndex(values[5], "argument-adapter"); + arguments.mft_index = ParseIndex(values[7], "argument-mft"); + arguments.profile.preference = ParseDegradationPreference(values[9]); + ParseOutputProfiles(arguments, 11, count, values); + arguments.profile = arguments.outputs.front(); + for (const auto& output : arguments.outputs) { + arguments.profile.width = std::max(arguments.profile.width, output.width); + arguments.profile.height = std::max(arguments.profile.height, output.height); + arguments.profile.frame_rate = std::max(arguments.profile.frame_rate, output.frame_rate); + arguments.profile.bit_rate = std::max(arguments.profile.bit_rate, output.bit_rate); + } + return arguments; + } else if (count >= 28 && count <= 23 + 5 * screener::capture::kMaxOutputs && (count - 23) % 5 == 0 && + std::wstring(values[1]) == L"--capture-video" && std::wstring(values[6]) == L"--adapter-index" && std::wstring(values[8]) == L"--mft-index" && std::wstring(values[10]) == L"--width" && @@ -1977,7 +1032,7 @@ ProductArguments ParseProductArguments(int count, wchar_t** values) { std::wstring(values[16]) == L"--bitrate" && std::wstring(values[18]) == L"--preference" && std::wstring(values[20]) == L"--codec" && - std::wstring(values[22]) == L"--protocol-v4") { + std::wstring(values[22]) == L"--protocol-v7") { arguments.mode = ProductArguments::Mode::video; arguments.target_kind = ParseTargetKind(values[2]); arguments.source_id = ParseUint64(values[3], "argument-source"); @@ -1994,6 +1049,12 @@ ProductArguments ParseProductArguments(int count, wchar_t** values) { } arguments.codec = NarrowAscii(codec); ValidateVideoProfile(arguments.profile); + ParseOutputProfiles(arguments, 23, count, values); + const auto& original = arguments.outputs[arguments.outputs.size() > 1 ? 1 : 0]; + if (original.width != arguments.profile.width || original.height != arguments.profile.height || + original.frame_rate != arguments.profile.frame_rate || original.bit_rate != arguments.profile.bit_rate) { + Fail("argument-output-source", "original output must match the source profile"); + } } else { Fail("arguments", "unsupported or incomplete command-line argument"); } @@ -2021,6 +1082,13 @@ HRESULT RunAudioCapture(const ProductArguments& arguments) { UniqueHandle stop(CreateEventW(nullptr, TRUE, FALSE, nullptr)); if (stop.get() == nullptr) return HRESULT_FROM_WIN32(GetLastError()); ProtocolWriter writer; + ControlReader controls; + auto should_stop = [&]() { + const auto commands = controls.Read(); + if (commands.empty()) return false; + if (commands.front().kind != 'Q') Fail("audio-control", "audio process only accepts stop"); + return true; + }; auto ready = [&writer]() { return writer.WriteStatus("{\"state\":\"active\",\"audio\":true}"); }; @@ -2031,11 +1099,11 @@ HRESULT RunAudioCapture(const ProductArguments& arguments) { }; if (arguments.target_kind == screener::capture::TargetKind::display) { return screener::capture::CaptureSystemAudio( - stop.get(), []() { return ConsumeControlSignal().stop; }, ready, pcm); + stop.get(), should_stop, ready, pcm); } return screener::capture::CaptureProcessAudio( arguments.pid, arguments.creation_time, stop.get(), - []() { return ConsumeControlSignal().stop; }, + should_stop, ready, pcm); } @@ -2078,7 +1146,7 @@ void WriteCapabilityProbe() { std::vector adapters = EnumerateAdapters(); std::ostringstream output; - output << "{\"protocol\":4,\"platform\":\"windows\",\"platformBuild\":" + output << "{\"protocol\":7,\"platform\":\"windows\",\"platformBuild\":" << JSONString(std::to_string(build)) << ",\"videoCapture\":" << (window_capture ? "true" : "false") << ",\"softwareVP8\":true" @@ -2118,19 +1186,46 @@ void WriteCapabilityProbe() { } } -double MeasureEncoderWork(VideoEncoder& encoder, ID3D11Device* device, +double MeasureEncoderWork(VideoEncoder* hardware, ID3D11Device* device, const VideoProfile& profile, EncoderClock::time_point deadline) { const UINT32 warmup_frames = (profile.frame_rate + 1) / 2; const UINT32 sample_frames = profile.frame_rate; + std::unique_ptr adaptive; + if (!hardware) { + adaptive = std::make_unique(OutputKind::vp8, profile, device, + AdaptiveEncoder::Factory{}); + } + const auto cadence_start = EncoderClock::now(); std::chrono::duration measured{}; for (UINT32 frame = 0; frame < warmup_frames + sample_frames; ++frame) { RequireEncoderTime(deadline); auto texture = CreateSyntheticTexture(device, frame, profile); + if (adaptive) { + // VSE observes real input cadence; pacing is not encoder work. + std::this_thread::sleep_until(cadence_start + std::chrono::nanoseconds( + static_cast(frame) * profile.frame_duration_100ns() * 100)); + RequireEncoderTime(deadline); + } const auto start = EncoderClock::now(); - auto encoded = encoder.Encode(texture.Get(), - (static_cast(frame) + 1) * profile.frame_duration_100ns(), - frame == 0, deadline); + const auto timestamp = (static_cast(frame) + 1) * profile.frame_duration_100ns(); + EncodedAccessUnit encoded; + if (adaptive) { + auto output = adaptive->Encode([&](UINT32 width, UINT32 height) { + if (width != profile.width || height != profile.height) + Fail("codec-probe-output", "encoder reduced the probe resolution"); + return texture; + }, profile.width, profile.height, timestamp, frame == 0, profile.bit_rate); + RequireEncoderTime(deadline); + if (!output || output->width != profile.width || output->height != profile.height || + output->duration100ns != static_cast(profile.frame_duration_100ns())) { + Fail("codec-probe-output", "encoder reduced the probe profile or dropped a frame"); + } + encoded = std::move(output->access_unit); + } else { + encoded = hardware->Encode(texture.Get(), timestamp, frame == 0, deadline); + } + // A dropped input cannot count as a throughput-probe output frame. if (encoded.bytes.empty() || encoded.bytes.size() > kMaxProductAccessUnitBytes) { Fail("codec-probe-output", "encoder produced an invalid probe frame"); } @@ -2139,46 +1234,417 @@ double MeasureEncoderWork(VideoEncoder& encoder, ID3D11Device* device, return measured.count() / sample_frames; } -std::unique_ptr SelectVideoEncoder( +struct VideoEncoderSelection final { + OutputKind kind = OutputKind::vp8; + std::unique_ptr initial; +}; + +VideoEncoderSelection SelectVideoEncoder( const ProductArguments& arguments, const Adapter& adapter, const DeviceContext& device) { - const auto create = [&](bool hardware) -> std::unique_ptr { - if (!hardware) { - return std::make_unique(device.device.Get(), arguments.profile); - } + const auto create = [&]() -> std::unique_ptr { ActivationList activations = EnumerateHardwareEncoders(adapter); auto selected = ActivateTransform(activations, arguments.mft_index, device.manager.Get()); return std::make_unique(std::move(selected), arguments.profile); }; - if (arguments.codec != "auto") return create(arguments.codec == "h264"); + if (arguments.codec == "vp8") return {}; + if (arguments.codec == "h264") return {OutputKind::h264, create()}; const auto began = EncoderClock::now(); const auto budget = std::chrono::seconds(4); const auto deadline = began + budget; std::optional hardware_work; try { - auto hardware = create(true); - hardware_work = MeasureEncoderWork(*hardware, device.device.Get(), + auto hardware = create(); + hardware_work = MeasureEncoderWork(hardware.get(), device.device.Get(), arguments.profile, began + budget / 2); - if (*hardware_work <= 1.0 / arguments.profile.frame_rate) return hardware; + if (*hardware_work <= 1.0 / arguments.profile.frame_rate) + return {OutputKind::h264, std::move(hardware)}; } catch (const std::exception&) { // No media has been published; a failed hardware probe may try software. } RequireEncoderTime(deadline); - std::unique_ptr software; try { - software = create(false); - const double software_work = MeasureEncoderWork(*software, device.device.Get(), + const double software_work = MeasureEncoderWork(nullptr, device.device.Get(), arguments.profile, deadline); - if (!hardware_work || software_work <= *hardware_work) return software; + if (!hardware_work || software_work <= *hardware_work) return {}; } catch (const std::exception&) { if (!hardware_work) throw; } - software.reset(); RequireEncoderTime(deadline); - auto hardware = create(true); + auto hardware = create(); RequireEncoderTime(deadline); - return hardware; + return {OutputKind::h264, std::move(hardware)}; +} + +struct CaptureInput final { + ComPtr texture; + UINT32 width = 0; + UINT32 height = 0; + SIZE presentation{}; + UINT64 timestamp = 0; +}; + +class OutputWorker final { + public: + using Factory = AdaptiveEncoder::Factory; + OutputWorker(UINT8 layer, VideoProfile profile, OutputKind kind, ID3D11Device* device, + ProtocolWriter& writer, Factory create, + std::unique_ptr initial, bool active, + std::function on_output, + std::function on_failure) + : layer_(layer), profile_(profile), kind_(kind), device_(device), writer_(writer), + create_(std::move(create)), initial_(std::move(initial)), + on_output_(std::move(on_output)), on_failure_(std::move(on_failure)), + mailbox_(profile.bit_rate, active), thread_([this]() { Run(); }) {} + + ~OutputWorker() { Stop(); Join(); } + + void Submit(std::shared_ptr input) { + mailbox_.Submit(std::move(input)); + } + + bool SetActive(bool active) { return mailbox_.SetActive(active); } + void RequestKeyFrame() { mailbox_.RequestKeyFrame(); } + void SetBitrate(UINT32 bitrate) { mailbox_.SetBitrate(bitrate); } + + void Stop() { + mailbox_.Stop(); + if (thread_.joinable()) CancelSynchronousIo(thread_.native_handle()); + } + + void Join() { + if (!thread_.joinable()) return; + // Keep cancellation effective if a retiring worker was about to enter WriteFile. + const DWORD interval = std::max(1, 1000 / profile_.frame_rate); + while (WaitForSingleObject(thread_.native_handle(), interval) == WAIT_TIMEOUT) { + CancelSynchronousIo(thread_.native_handle()); + } + thread_.join(); + } + + private: + void Run() noexcept { + bool apartment = false; + bool recovery = false; + auto initial = std::move(initial_); + std::unique_ptr encoder; + std::unique_ptr converter; + try { + Check(CoInitializeEx(nullptr, COINIT_MULTITHREADED), "output-com-apartment"); + apartment = true; + UINT64 codec_generation = 0; + UINT32 converted_width = 0, converted_height = 0; + for (;;) { + const auto work = mailbox_.Take(); + if (work.action == Mailbox::Action::stop) break; + if (work.action == Mailbox::Action::retire) { + encoder.reset(); + initial.reset(); + converter.reset(); + continue; + } + const auto& input = work.input; + recovery = work.recovery; + if (codec_generation != work.generation) { + encoder.reset(); + initial.reset(); + converter.reset(); + codec_generation = work.generation; + } + if (!encoder) { + encoder = std::make_unique( + kind_, profile_, device_.Get(), create_, std::move(initial)); + } + auto output = encoder->Encode([&](UINT32 width, UINT32 height) { + if (!converter || converted_width != width || converted_height != height) { + auto selected = profile_; + selected.width = width; + selected.height = height; + converter = std::make_unique(device_.Get(), selected); + converted_width = width; + converted_height = height; + } + return converter->Convert(input->texture.Get(), input->width, + input->height, input->presentation); + }, profile_.width, profile_.height, input->timestamp, recovery, work.bitrate); + if (!output) continue; + if (!mailbox_.Accept(work.generation)) continue; + const auto& access_unit = output->access_unit; + Check(writer_.Write(kind_, access_unit.key_frame ? 1 : 0, + access_unit.timestamp100ns, output->duration100ns, + access_unit.bytes.data(), static_cast(access_unit.bytes.size()), + layer_, static_cast(output->width), + static_cast(output->height)), "capture-video-output"); + if (on_output_) on_output_(); + } + } catch (...) { + if (mailbox_.Fail(recovery)) on_failure_(std::current_exception()); + } + encoder.reset(); + initial.reset(); + converter.reset(); + if (apartment) CoUninitialize(); + } + + const UINT8 layer_; + const VideoProfile profile_; + const OutputKind kind_; + ComPtr device_; + ProtocolWriter& writer_; + Factory create_; + std::unique_ptr initial_; + std::function on_output_; + std::function on_failure_; + using Mailbox = screener::capture::OutputMailbox; + Mailbox mailbox_; + std::thread thread_; +}; + +void AppendOutputProfiles(std::ostream& output, const std::vector& profiles) { + output << ",\"outputs\":["; + for (size_t index = 0; index < profiles.size(); ++index) { + if (index != 0) output << ','; + const auto& profile = profiles[index]; + output << "{\"width\":" << profile.width << ",\"height\":" << profile.height + << ",\"fps\":" << profile.frame_rate << ",\"bitrate\":" << profile.bit_rate << '}'; + } + output << ']'; +} + +void WriteVideoStarting(ProtocolWriter& writer, const ProductArguments& arguments, + const Adapter& adapter, const VideoEncoderSelection& encoder) { + const bool hardware = encoder.kind == OutputKind::h264; + std::ostringstream status; + status << "{\"state\":\"starting\",\"hardwareOnly\":" << (hardware ? "true" : "false") + << ",\"codec\":" << JSONString(hardware ? "h264" : "vp8") + << ",\"adapterIndex\":" << adapter.index + << ",\"adapterName\":" << JSONString(NarrowAscii(adapter.description.Description)) + << ",\"adapterIdentity\":" << JSONString(LuidString(adapter.description.AdapterLuid)) + << ",\"encoderName\":" << JSONString(hardware ? encoder.initial->name : "libvpx VP8") + << ",\"encoderIdentity\":" << JSONString(hardware ? encoder.initial->identity : + std::string("libvpx/") + vpx_codec_version_str()); + if (hardware) status << ",\"encoderIndex\":" << arguments.mft_index; + AppendOutputProfiles(status, arguments.outputs); + status << '}'; + Check(writer.WriteStatus(status.str()), "capture-status-starting"); +} + +void WriteVideoActive(ProtocolWriter& writer, const ProductArguments& arguments, bool hardware) { + std::ostringstream status; + status << "{\"state\":\"active\",\"hardwareOnly\":" << (hardware ? "true" : "false") + << ",\"codec\":" << JSONString(hardware ? "h264" : "vp8"); + if (hardware) status << ",\"profileLevelId\":" << JSONString(arguments.profile.profile_level_id()); + status << ",\"width\":" << arguments.profile.width << ",\"height\":" << arguments.profile.height + << ",\"fps\":" << arguments.profile.frame_rate; + AppendOutputProfiles(status, arguments.outputs); + status << '}'; + Check(writer.WriteStatus(status.str()), "capture-status-active"); +} + +std::vector> CreateOutputWorkers( + const ProductArguments& arguments, const Adapter& adapter, const DeviceContext& device, + ProtocolWriter& writer, VideoEncoderSelection encoder, + const std::function& on_active, + const std::function& on_failure) { + const bool hardware = encoder.kind == OutputKind::h264; + const UINT encoder_index = arguments.mft_index; + const bool capture = arguments.mode == ProductArguments::Mode::video; + const size_t original_layer = capture && arguments.outputs.size() > 1 ? 1 : 0; + std::vector> workers; + for (size_t layer = 0; layer < arguments.outputs.size(); ++layer) { + const auto profile = arguments.outputs[layer]; + const bool original = layer == original_layer; + AdaptiveEncoder::Factory create; + if (hardware) { + create = [&adapter, &device, encoder_index](const VideoProfile& selected_profile) -> std::unique_ptr { + auto activations = EnumerateHardwareEncoders(adapter); + auto selected = ActivateTransform(activations, encoder_index, device.manager.Get()); + return std::make_unique(std::move(selected), selected_profile); + }; + } + workers.push_back(std::make_unique( + static_cast(layer), profile, encoder.kind, device.device.Get(), writer, + std::move(create), original ? std::move(encoder.initial) : nullptr, capture && layer < 2, + [original, on_active]() { if (original) on_active(); }, + [layer, on_failure](std::exception_ptr error) { on_failure(layer, error); })); + } + return workers; +} + +std::string OutputFailureDetail(std::exception_ptr error) { + try { + std::rethrow_exception(error); + } catch (const GateFailure& failed) { + return failed.stage(); + } catch (const std::exception& failed) { + return failed.what(); + } catch (...) { + return "Output encoder is unavailable"; + } +} + +bool ApplyOutputControl(const screener::capture::CaptureControl& control, + const std::vector>& workers) { + if (control.layer >= static_cast(workers.size())) Fail("control-layer", "output layer is unavailable"); + if (control.kind == 'A') { + return workers[control.layer]->SetActive(control.value != 0); + } else if (control.kind == 'K') { + for (size_t layer = 0; layer < workers.size(); ++layer) { + if (control.layer == -1 || static_cast(control.layer) == layer) workers[layer]->RequestKeyFrame(); + } + } else if (control.kind == 'B') { + workers[control.layer]->SetBitrate(control.value); + } + return false; +} + +ComPtr UploadDecodedNV12(const DeviceContext& device, + const BYTE* data, UINT32 width, UINT32 height) { + D3D11_TEXTURE2D_DESC description{}; + description.Width = width; + description.Height = height; + description.MipLevels = description.ArraySize = 1; + description.Format = DXGI_FORMAT_NV12; + description.SampleDesc.Count = 1; + description.Usage = D3D11_USAGE_DEFAULT; + description.BindFlags = D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_RENDER_TARGET; + D3D11_SUBRESOURCE_DATA input{}; + input.pSysMem = data; + input.SysMemPitch = width; + input.SysMemSlicePitch = width * height * 3 / 2; + ComPtr texture; + Check(device.device->CreateTexture2D(&description, &input, &texture), "decoded-input-upload"); + return texture; +} + +ComPtr OwnDecodedTexture(const DeviceContext& device, IMFSample* sample, + UINT32 width, UINT32 height, LONG stride) { + ComPtr buffer; + Check(sample->GetBufferByIndex(0, &buffer), "decoded-buffer"); + ComPtr dxgi; + if (SUCCEEDED(buffer.As(&dxgi))) { + ComPtr original; + UINT subresource = 0; + Check(dxgi->GetResource(IID_PPV_ARGS(&original)), "decoded-dxgi-resource"); + Check(dxgi->GetSubresourceIndex(&subresource), "decoded-dxgi-subresource"); + D3D11_TEXTURE2D_DESC description{}; + original->GetDesc(&description); + if (description.Format != DXGI_FORMAT_NV12 || description.Width < width || description.Height < height || + description.Width > 2560 || description.Height > 1440) Fail("decoded-texture-size", "decoder texture exceeds source bounds"); + description.ArraySize = description.MipLevels = 1; + description.Usage = D3D11_USAGE_DEFAULT; + description.CPUAccessFlags = description.MiscFlags = 0; + description.BindFlags = D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_RENDER_TARGET; + ComPtr owned; + Check(device.device->CreateTexture2D(&description, nullptr, &owned), "decoded-owned-texture"); + device.context->CopySubresourceRegion(owned.Get(), 0, 0, 0, 0, original.Get(), subresource, nullptr); + return owned; + } + const DWORD size = width * height * 3 / 2; + std::vector pixels(size); + ComPtr planar; + if (SUCCEEDED(buffer.As(&planar))) { + DWORD length = 0; + Check(planar->GetContiguousLength(&length), "decoded-contiguous-length"); + if (length != size) Fail("decoded-contiguous-size", "NV12 output does not match its dimensions"); + Check(planar->ContiguousCopyTo(pixels.data(), size), "decoded-contiguous-copy"); + } else { + BYTE* data = nullptr; + DWORD length = 0; + Check(buffer->Lock(&data, nullptr, &length), "decoded-memory-lock"); + const bool valid = stride >= static_cast(width) && + static_cast(stride) * height * 3 / 2 <= length; + if (valid) { + for (UINT32 row = 0; row < height * 3 / 2; ++row) { + std::copy_n(data + static_cast(row) * stride, width, pixels.data() + static_cast(row) * width); + } + } + Check(buffer->Unlock(), "decoded-memory-unlock"); + if (!valid) Fail("decoded-memory-size", "NV12 output buffer is incomplete"); + } + return UploadDecodedNV12(device, pixels.data(), width, height); +} + +void RunEncodedVideo(const ProductArguments& arguments) { + auto adapters = EnumerateAdapters(); + const auto& adapter = SelectAdapter(adapters, arguments.adapter_index); + const auto device = CreateDevice(adapter); + auto encoder = SelectVideoEncoder(arguments, adapter, device); + const bool hardware = arguments.codec == "h264"; + ProtocolWriter writer; + WriteVideoStarting(writer, arguments, adapter, encoder); + HANDLE input_thread_handle = nullptr; + Check(DuplicateHandle(GetCurrentProcess(), GetCurrentThread(), GetCurrentProcess(), + &input_thread_handle, 0, FALSE, DUPLICATE_SAME_ACCESS), "encoded-input-thread"); + UniqueHandle input_thread(input_thread_handle); + std::atomic active{false}; + std::atomic failed{0}; + auto workers = CreateOutputWorkers(arguments, adapter, device, writer, std::move(encoder), + [&]() { if (!active.exchange(true)) WriteVideoActive(writer, arguments, hardware); }, + [&](size_t layer, std::exception_ptr error) { + (void)writer.WriteUnavailable(static_cast(layer), OutputFailureDetail(error)); + if (failed.fetch_add(1) + 1 == arguments.outputs.size()) CancelSynchronousIo(input_thread.get()); + }); + auto cleanup = [&]() { + writer.Stop(); + for (auto& worker : workers) worker->Stop(); + for (auto& worker : workers) worker->Join(); + }; + try { + std::unique_ptr h264; + std::unique_ptr vp8; + if (hardware) h264 = std::make_unique(device.manager.Get()); + else vp8 = std::make_unique(); + InputReader reader; + bool began = false; + UINT64 last_timestamp = 0; + auto submit = [&](ComPtr texture, UINT32 width, UINT32 height, + UINT64 timestamp, UINT64 duration) { + if ((began && timestamp <= last_timestamp) || width < arguments.profile.width || height < arguments.profile.height) { + Fail("encoded-input-output", "decoded frame changed the source timeline or output bounds"); + } + auto input = std::make_shared(); + input->texture = std::move(texture); + input->width = width; + input->height = height; + input->presentation = {static_cast(width), static_cast(height)}; + input->timestamp = timestamp; + Check(writer.WriteBegin(timestamp, duration), "decoded-frame-begin"); + for (auto& worker : workers) worker->Submit(input); + began = true; + last_timestamp = timestamp; + }; + for (;;) { + if (failed.load() == workers.size()) Fail("encoded-outputs", "all derived outputs are unavailable"); + for (const auto& input : reader.Read(true)) { + if (input.kind == 7) { + const auto control = input.Control(); + if (control.kind == 'Q') { cleanup(); return; } + ApplyOutputControl(control, workers); + continue; + } + if (input.kind != (hardware ? 2 : 4)) Fail("encoded-input-codec", "encoded source codec changed"); + if (h264) { + h264->Decode(input, [&](IMFSample* sample, UINT32 width, UINT32 height, LONG stride) { + LONGLONG timestamp = -1, duration = 0; + Check(sample->GetSampleTime(×tamp), "decoded-timestamp"); + Check(sample->GetSampleDuration(&duration), "decoded-duration"); + if (timestamp < 0 || duration <= 0) Fail("decoded-timing", "decoder did not preserve source timing"); + submit(OwnDecodedTexture(device, sample, width, height, stride), width, height, + static_cast(timestamp), static_cast(duration)); + }); + } else { + const auto decoded = vp8->Decode(input.data); + if (!decoded.nv12.empty()) submit(UploadDecodedNV12(device, decoded.nv12.data(), decoded.width, decoded.height), + decoded.width, decoded.height, input.timestamp, input.duration); + } + } + } + } catch (...) { + cleanup(); + throw; + } } void RunVideoCapture(const ProductArguments& arguments) { @@ -2196,9 +1662,7 @@ void RunVideoCapture(const ProductArguments& arguments) { const Adapter& adapter = SelectAdapter(adapters, arguments.adapter_index); DeviceContext device = CreateDevice(adapter); auto encoder = SelectVideoEncoder(arguments, adapter, device); - const bool hardware = encoder->kind == OutputKind::h264; - const char* codec = hardware ? "h264" : "vp8"; - FrameConverter converter(device.device.Get(), arguments.profile); + const bool hardware = encoder.kind == OutputKind::h264; screener::capture::CapturePresentation presentation(arguments.target_kind, arguments.source_id); ProtocolWriter writer; @@ -2258,8 +1722,21 @@ void RunVideoCapture(const ProductArguments& arguments) { item_closed.store(true); SetEvent(event); }); + std::vector> workers; + std::atomic active_status_written{false}; + std::mutex failure_mutex; + std::exception_ptr failure; + auto fail_capture = [&](std::exception_ptr error) { + { + std::lock_guard lock(failure_mutex); + if (!failure) failure = error; + } + SetEvent(shutdown.get()); + }; auto cleanup = [&]() noexcept { SetEvent(shutdown.get()); + writer.Stop(); + for (auto& worker : workers) worker->Stop(); try { pool.FrameArrived(frame_token); } catch (...) { @@ -2276,73 +1753,60 @@ void RunVideoCapture(const ProductArguments& arguments) { pool.Close(); } catch (...) { } + for (auto& worker : workers) worker->Join(); }; try { const UINT64 frame_duration = static_cast( arguments.profile.frame_duration_100ns()); - const UINT64 envelope_frame_duration = (frame_duration / 10) * 10; - std::ostringstream starting; - starting << "{\"state\":\"starting\",\"hardwareOnly\":" - << (hardware ? "true" : "false") << ",\"codec\":" << JSONString(codec) - << ",\"adapterIndex\":" << adapter.index - << ",\"adapterName\":" - << JSONString(NarrowAscii(adapter.description.Description)) - << ",\"adapterIdentity\":" - << JSONString(LuidString(adapter.description.AdapterLuid)) - << ",\"encoderName\":" - << JSONString(encoder->name) - << ",\"encoderIdentity\":" - << JSONString(encoder->identity); - if (hardware) starting << ",\"encoderIndex\":" << arguments.mft_index; - starting << '}'; - Check(writer.WriteStatus(starting.str()), "capture-status-starting"); + const DWORD control_wait_ms = static_cast((frame_duration + 9'999) / 10'000); + WriteVideoStarting(writer, arguments, adapter, encoder); + workers = CreateOutputWorkers(arguments, adapter, device, writer, std::move(encoder), + [&]() { + if (!active_status_written.exchange(true)) WriteVideoActive(writer, arguments, hardware); + }, + [&](size_t layer, std::exception_ptr error) { + if (layer == (arguments.outputs.size() > 1 ? 1u : 0u)) { + fail_capture(error); + return; + } + if (FAILED(writer.WriteUnavailable(static_cast(layer), OutputFailureDetail(error)))) fail_capture(error); + }); capture_session.StartCapture(); UINT64 previous_timestamp = 0; UINT64 next_output_timestamp = 0; - UINT64 encoded_frames = 0; - bool active_status_written = false; + UINT64 source_frames = 0; auto pool_size = initial_size; - ComPtr latest_nv12; - UINT64 latest_timestamp = 0; - bool pending_key_frame = false; - auto write_encoded = [&](ID3D11Texture2D* texture, UINT64 timestamp, - bool force_key_frame) { - EncodedAccessUnit access_unit = - encoder->Encode(texture, timestamp, force_key_frame); - if (access_unit.bytes.size() > kMaxProductAccessUnitBytes) { - Fail("output-buffer-bounds", "live video access unit exceeded one MiB"); - } - UINT64 wire_timestamp = (access_unit.timestamp100ns / 10) * 10; - Check(writer.Write(encoder->kind, access_unit.key_frame ? 1 : 0, - wire_timestamp, envelope_frame_duration, - access_unit.bytes.data(), - static_cast(access_unit.bytes.size())), - "capture-video-output"); - previous_timestamp = timestamp; - ++encoded_frames; - if (!active_status_written) { - std::ostringstream active; - active << "{\"state\":\"active\",\"hardwareOnly\":" - << (hardware ? "true" : "false") << ",\"codec\":" << JSONString(codec); - if (hardware) { - active << ",\"profileLevelId\":" << JSONString(arguments.profile.profile_level_id()); - } - active << ",\"width\":" << arguments.profile.width << ",\"height\":" - << arguments.profile.height << ",\"fps\":" - << arguments.profile.frame_rate << '}'; - Check(writer.WriteStatus(active.str()), "capture-status-active"); - active_status_written = true; - } + std::shared_ptr latest_input; + bool refresh_input = false; + ControlReader controls; + auto submit = [&](std::shared_ptr input) { + Check(writer.WriteBegin(input->timestamp, frame_duration), "capture-frame-begin"); + for (auto& worker : workers) worker->Submit(input); + previous_timestamp = input->timestamp; + ++source_frames; + refresh_input = false; }; const HANDLE window_waits[] = { process.get(), shutdown.get(), frame_ready.get()}; const HANDLE display_waits[] = {shutdown.get(), frame_ready.get()}; for (;;) { + for (const auto& control : controls.Read()) { + if (control.kind == 'Q') { + cleanup(); + return; + } + if (control.kind == 'A') { + refresh_input = ApplyOutputControl(control, workers) || refresh_input; + continue; + } + ApplyOutputControl(control, workers); + refresh_input = true; + } DWORD wait = window_target - ? WaitForMultipleObjects(3, window_waits, FALSE, 5'000) - : WaitForMultipleObjects(2, display_waits, FALSE, 5'000); + ? WaitForMultipleObjects(3, window_waits, FALSE, control_wait_ms) + : WaitForMultipleObjects(2, display_waits, FALSE, control_wait_ms); if (window_target && wait == WAIT_OBJECT_0) { Fail("target-exited", "selected target process exited"); } @@ -2351,27 +1815,21 @@ void RunVideoCapture(const ProductArguments& arguments) { const DWORD frame_index = window_target ? WAIT_OBJECT_0 + 2 : WAIT_OBJECT_0 + 1; if (wait == shutdown_index) { + { + std::lock_guard lock(failure_mutex); + if (failure) std::rethrow_exception(failure); + } if (item_closed.load()) { Fail("capture-closed", "selected source stopped capture"); } Fail("capture-stopped", "source capture stopped"); } if (wait == WAIT_TIMEOUT) { - // A quiet source is not a capture-end signal. The item/process wait - // handles remain the terminal signals; a pending keyframe can reuse - // the latest converted image. - ControlSignal control = ConsumeControlSignal(); - if (control.stop) { - cleanup(); - return; - } - pending_key_frame = pending_key_frame || control.key_frame; - if (pending_key_frame && latest_nv12) { - const UINT64 timestamp = previous_timestamp == 0 - ? latest_timestamp - : previous_timestamp + frame_duration; - write_encoded(latest_nv12.Get(), timestamp, true); - pending_key_frame = false; + // An explicit output/control request may reuse a quiet source; silence is not EOF. + if (refresh_input && latest_input) { + auto input = std::make_shared(*latest_input); + input->timestamp = previous_timestamp + frame_duration; + submit(input); } continue; } @@ -2403,8 +1861,7 @@ void RunVideoCapture(const ProductArguments& arguments) { content_size.Height != pool_size.Height) { latest.Close(); latest = nullptr; - latest_nv12 = nullptr; - latest_timestamp = 0; + latest_input.reset(); pool.Recreate(capture_device, DirectXPixelFormat::B8G8R8A8UIntNormalized, 2, content_size); @@ -2418,10 +1875,21 @@ void RunVideoCapture(const ProductArguments& arguments) { source_description.Width, static_cast(content_size.Width)); UINT32 content_height = std::min( source_description.Height, static_cast(content_size.Height)); - ComPtr nv12 = converter.Convert( - source.Get(), content_width, content_height, - presentation.Resolve(content_width, content_height, - encoded_frames % arguments.profile.gop_frames() == 0)); + auto input = std::make_shared(); + auto owned_description = source_description; + owned_description.Usage = D3D11_USAGE_DEFAULT; + owned_description.CPUAccessFlags = 0; + owned_description.MiscFlags = 0; + owned_description.BindFlags = D3D11_BIND_SHADER_RESOURCE | D3D11_BIND_RENDER_TARGET; + Check(device.device->CreateTexture2D(&owned_description, nullptr, &input->texture), "capture-owned-input"); + device.context->CopyResource(input->texture.Get(), source.Get()); + input->width = content_width; + input->height = content_height; + input->timestamp = timestamp; + input->presentation = presentation.Resolve(content_width, content_height, + source_frames % arguments.profile.gop_frames() == 0); + latest.Close(); + latest = nullptr; if (next_output_timestamp == 0) { next_output_timestamp = timestamp + frame_duration; } else { @@ -2431,19 +1899,8 @@ void RunVideoCapture(const ProductArguments& arguments) { next_output_timestamp += (elapsed / frame_duration + 1) * frame_duration; } - latest_nv12 = nv12; - latest_timestamp = timestamp; - ControlSignal control = ConsumeControlSignal(); - if (control.stop) { - cleanup(); - return; - } - pending_key_frame = pending_key_frame || control.key_frame; - bool key_frame = encoded_frames == 0 || - encoded_frames % arguments.profile.gop_frames() == 0 || - pending_key_frame; - write_encoded(nv12.Get(), timestamp, key_frame); - pending_key_frame = false; + latest_input = input; + submit(input); } } catch (...) { cleanup(); @@ -2594,7 +2051,8 @@ int wmain(int argc, wchar_t** argv) { return 0; } Runtime runtime; - RunVideoCapture(arguments); + if (arguments.mode == ProductArguments::Mode::encoded) RunEncodedVideo(arguments); + else RunVideoCapture(arguments); return 0; } catch (const GateFailure& error) { std::cerr << "result=window-capture-failed\n" diff --git a/native/capture/windows/output_mailbox.h b/native/capture/windows/output_mailbox.h new file mode 100644 index 0000000..86a4997 --- /dev/null +++ b/native/capture/windows/output_mailbox.h @@ -0,0 +1,108 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace screener::capture { + +// One codec worker consumes raw inputs; only its not-yet-encoded input is replaceable. +template +class OutputMailbox final { + public: + enum class Action { frame, retire, stop }; + struct Work final { + Action action = Action::stop; + std::shared_ptr input; + uint64_t generation = 0; + uint32_t bitrate = 0; + bool recovery = false; + }; + + explicit OutputMailbox(uint32_t ceiling, bool active = true) + : ceiling_(ceiling), bitrate_(ceiling), active_(active) {} + + void Submit(std::shared_ptr input) { + std::lock_guard lock(mutex_); + if (active_ && !stopped_ && !failed_) { + pending_ = std::move(input); + changed_.notify_one(); + } + } + + Work Take() { + std::unique_lock lock(mutex_); + changed_.wait(lock, [&]() { return stopped_ || retire_ || pending_; }); + if (stopped_) return {}; + if (retire_) { + retire_ = false; + return {Action::retire, {}, generation_, bitrate_, false}; + } + return {Action::frame, std::exchange(pending_, {}), generation_, bitrate_, + std::exchange(key_requested_, false)}; + } + + // Only a fresh activation needs a retained capture frame. + bool SetActive(bool active) { + std::lock_guard lock(mutex_); + if (active_ == active || failed_) return false; + active_ = active; + ++generation_; + key_requested_ = true; + if (!active) { + pending_.reset(); + retire_ = true; + } + changed_.notify_one(); + return active; + } + + void RequestKeyFrame() { + std::lock_guard lock(mutex_); + key_requested_ = true; + } + + void SetBitrate(uint32_t bitrate) { + if (bitrate < 1'000 || bitrate > ceiling_) throw std::runtime_error("Output bitrate exceeds its codec bounds"); + std::lock_guard lock(mutex_); + bitrate_ = bitrate; + } + + bool Accept(uint64_t generation) { + std::lock_guard lock(mutex_); + return !stopped_ && !failed_ && active_ && generation == generation_; + } + + bool Fail(bool recovery) { + std::lock_guard lock(mutex_); + key_requested_ = key_requested_ || recovery; + failed_ = true; + pending_.reset(); + return !stopped_; + } + + void Stop() { + std::lock_guard lock(mutex_); + stopped_ = true; + pending_.reset(); + changed_.notify_one(); + } + + private: + const uint32_t ceiling_; + std::mutex mutex_; + std::condition_variable changed_; + std::shared_ptr pending_; + uint32_t bitrate_; + uint64_t generation_ = 0; + bool active_ = true; + bool stopped_ = false; + bool failed_ = false; + bool retire_ = false; + bool key_requested_ = true; +}; + +} // namespace screener::capture diff --git a/native/capture/windows/vp8_decoder.h b/native/capture/windows/vp8_decoder.h new file mode 100644 index 0000000..584ed58 --- /dev/null +++ b/native/capture/windows/vp8_decoder.h @@ -0,0 +1,75 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include + +namespace screener::capture { + +// Decoder image storage belongs to libvpx until the next Decode call. A single +// NV12 copy is shared by all derived-output workers after upload. +class Vp8Decoder final { + public: + struct Image final { + uint32_t width = 0; + uint32_t height = 0; + std::vector nv12; + }; + + Vp8Decoder() { + vpx_codec_dec_cfg_t config{}; + config.threads = 2; + if (vpx_codec_dec_init(&codec_, vpx_codec_vp8_dx(), &config, 0) != VPX_CODEC_OK) { + throw std::runtime_error("VP8 decoder initialization failed"); + } + } + ~Vp8Decoder() { vpx_codec_destroy(&codec_); } + Vp8Decoder(const Vp8Decoder&) = delete; + Vp8Decoder& operator=(const Vp8Decoder&) = delete; + + Image Decode(const std::vector& data) { + if (data.empty() || data.size() > 4 * 1024 * 1024) throw std::runtime_error("Invalid VP8 decoder input size"); + if (!(data[0] & 1)) { + vpx_codec_stream_info_t info{}; + info.sz = sizeof(info); + if (vpx_codec_peek_stream_info(vpx_codec_vp8_dx(), data.data(), static_cast(data.size()), &info) != VPX_CODEC_OK || + info.w < 2 || info.h < 2 || info.w > 2560 || info.h > 1440 || (info.w & 1) || (info.h & 1)) { + throw std::runtime_error("VP8 decoder input dimensions exceed the source bound"); + } + } + if (vpx_codec_decode(&codec_, data.data(), static_cast(data.size()), nullptr, 0) != VPX_CODEC_OK) { + throw std::runtime_error("VP8 decoder rejected its input"); + } + vpx_codec_iter_t iterator = nullptr; + const auto* image = vpx_codec_get_frame(&codec_, &iterator); + if (!image) return {}; + const uint32_t width = image->d_w, height = image->d_h; + if (width < 2 || height < 2 || width > 2560 || height > 1440 || (width & 1) || (height & 1) || + image->fmt != VPX_IMG_FMT_I420 || image->stride[0] < static_cast(width) || + image->stride[1] < static_cast(width / 2) || image->stride[2] < static_cast(width / 2)) { + throw std::runtime_error("VP8 decoder output format is unsupported"); + } + Image result{width, height, std::vector(static_cast(width) * height * 3 / 2)}; + for (uint32_t row = 0; row < height; ++row) { + std::copy_n(image->planes[0] + row * image->stride[0], width, result.nv12.data() + row * width); + } + auto* chroma = result.nv12.data() + static_cast(width) * height; + for (uint32_t row = 0; row < height / 2; ++row) { + for (uint32_t column = 0; column < width / 2; ++column) { + chroma[row * width + 2 * column] = image->planes[1][row * image->stride[1] + column]; + chroma[row * width + 2 * column + 1] = image->planes[2][row * image->stride[2] + column]; + } + } + if (vpx_codec_get_frame(&codec_, &iterator)) throw std::runtime_error("VP8 decoder emitted multiple frames for one access unit"); + return result; + } + + private: + vpx_codec_ctx_t codec_{}; +}; + +} // namespace screener::capture diff --git a/native/capture/windows/vp8_encoder.cpp b/native/capture/windows/vp8_encoder.cpp index 8a0a8dc..8fcd26e 100644 --- a/native/capture/windows/vp8_encoder.cpp +++ b/native/capture/windows/vp8_encoder.cpp @@ -28,7 +28,7 @@ Vp8Encoder::Vp8Encoder(uint32_t width, uint32_t height, uint32_t frame_rate, throw std::runtime_error("VP8 encoder profile is invalid"); } duration100ns_ = 10'000'000 / frame_rate; - vpx_codec_enc_cfg_t config{}; + auto& config = config_; CheckVpx(vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &config, 0)); config.g_w = width; config.g_h = height; @@ -57,6 +57,14 @@ Vp8Encoder::~Vp8Encoder() { vpx_codec_destroy(&codec_); } const char* Vp8Encoder::Version() { return vpx_codec_version_str(); } +void Vp8Encoder::SetBitrate(uint32_t bit_rate) { + if (bit_rate < 1'000) throw std::runtime_error("VP8 bitrate is invalid"); + auto updated = config_; + updated.rc_target_bitrate = bit_rate / 1'000; + CheckVpx(vpx_codec_enc_config_set(&codec_, &updated)); + config_ = updated; +} + Vp8Frame Vp8Encoder::Encode(uint8_t* nv12, size_t row_pitch, uint64_t timestamp100ns, bool key_frame) { if (nv12 == nullptr || row_pitch < width_ || @@ -89,7 +97,7 @@ Vp8Frame Vp8Encoder::Encode(uint8_t* nv12, size_t row_pitch, frame.bytes.assign(bytes, bytes + packet->data.frame.sz); frame.key_frame = (packet->data.frame.flags & VPX_FRAME_IS_KEY) != 0; } - if (frame.bytes.empty() || (key_frame && !frame.key_frame)) { + if (!frame.bytes.empty() && key_frame && !frame.key_frame) { throw std::runtime_error("VP8 encoder did not produce the requested frame"); } return frame; diff --git a/native/capture/windows/vp8_encoder.h b/native/capture/windows/vp8_encoder.h index bf3b28f..e266136 100644 --- a/native/capture/windows/vp8_encoder.h +++ b/native/capture/windows/vp8_encoder.h @@ -21,12 +21,15 @@ class Vp8Encoder final { Vp8Encoder(const Vp8Encoder&) = delete; Vp8Encoder& operator=(const Vp8Encoder&) = delete; + // An empty result is a codec-owned frame drop, not an encoder failure. Vp8Frame Encode(uint8_t* nv12, size_t row_pitch, uint64_t timestamp100ns, bool key_frame); + void SetBitrate(uint32_t bit_rate); static const char* Version(); private: vpx_codec_ctx_t codec_{}; + vpx_codec_enc_cfg_t config_{}; uint32_t width_; uint32_t height_; unsigned long duration100ns_; diff --git a/native/capture/windows/webrtc-dependencies.json b/native/capture/windows/webrtc-dependencies.json new file mode 100644 index 0000000..c3fd5ef --- /dev/null +++ b/native/capture/windows/webrtc-dependencies.json @@ -0,0 +1,19 @@ +{ + "webrtc": { + "version": "m152.7977.0.2", + "commit": "6f37672d358475cd17544121a12494da454d85fb", + "url": "https://github.com/shiguredo-webrtc-build/webrtc-build/releases/download/m152.7977.0.2/webrtc.windows_x86_64.zip", + "sha256": "3250ee091eb745b9885482d92db352e6bcd5426940021ff90a095747d84d2137" + }, + "msvc": { + "directory": "Contents/VC/Tools/MSVC/14.44.35207", + "crt": { + "url": "https://download.visualstudio.microsoft.com/download/pr/67cf767c-5e71-47c2-a54a-cd5631e28942/f01f701a7bcd9587a340898c851424f6a52bb913a70c185ff0d5bf0288c5831a/Microsoft.VC.14.44.17.14.CRT.x64.Desktop.base.vsix", + "sha256": "f01f701a7bcd9587a340898c851424f6a52bb913a70c185ff0d5bf0288c5831a" + }, + "tools": { + "url": "https://download.visualstudio.microsoft.com/download/pr/bbc72d8e-2acd-4229-8f6a-85e23c5e3456/ee0baaa3a112d255f19f6c27dcc0ff6e496949eb9f1f37be0ac908c562a7076c/Microsoft.VC.14.44.17.14.Tools.HostX64.TargetX64.base.vsix", + "sha256": "ee0baaa3a112d255f19f6c27dcc0ff6e496949eb9f1f37be0ac908c562a7076c" + } + } +} diff --git a/native/fixtures/mf-h264-hardware/build.ps1 b/native/fixtures/mf-h264-hardware/build.ps1 index d0a0da0..1e90a9e 100644 --- a/native/fixtures/mf-h264-hardware/build.ps1 +++ b/native/fixtures/mf-h264-hardware/build.ps1 @@ -28,15 +28,19 @@ if ([string]::IsNullOrWhiteSpace($installationPath)) { } $developerCommand = Join-Path $installationPath 'Common7\Tools\VsDevCmd.bat' -$sourcePath = Join-Path $fixtureDirectory '..\..\client\platform\windows\capture\main.cpp' +$sourcePath = Join-Path $fixtureDirectory '..\..\capture\windows\main.cpp' +$encoderSourcePath = Join-Path $fixtureDirectory '..\..\capture\windows\h264_encoder.cpp' $executablePath = Join-Path $outputPath 'screener-mf-h264-fixture.exe' $objectPath = Join-Path $outputPath 'main.obj' +$encoderObjectPath = Join-Path $outputPath 'h264-encoder.obj' New-Item -ItemType Directory -Path $outputPath -Force | Out-Null $compile = @( 'call "{0}" -arch=x64 -host_arch=x64 >nul' -f $developerCommand - 'cl.exe /nologo /std:c++20 /EHsc /W4 /WX /DSCREENER_H264_FIXTURE /DUNICODE /D_UNICODE /DWIN32_LEAN_AND_MEAN /D_WIN32_WINNT=0x0A00 "{0}" /Fo:"{1}" /Fe:"{2}" /link mfplat.lib mf.lib mfuuid.lib d3d11.lib dxgi.lib dxguid.lib evr.lib ole32.lib oleaut32.lib pdh.lib windowsapp.lib' -f $sourcePath, $objectPath, $executablePath + 'cl.exe /nologo /c /std:c++20 /EHsc /W4 /WX /DSCREENER_H264_FIXTURE /DUNICODE /D_UNICODE /DWIN32_LEAN_AND_MEAN /D_WIN32_WINNT=0x0A00 "{0}" /Fo:"{1}"' -f $sourcePath, $objectPath + 'cl.exe /nologo /c /std:c++20 /EHsc /W4 /WX /DUNICODE /D_UNICODE /DWIN32_LEAN_AND_MEAN /D_WIN32_WINNT=0x0A00 "{0}" /Fo:"{1}"' -f $encoderSourcePath, $encoderObjectPath + 'link.exe /nologo /out:"{0}" "{1}" "{2}" mfplat.lib mf.lib mfuuid.lib d3d11.lib dxgi.lib dxguid.lib evr.lib ole32.lib oleaut32.lib pdh.lib windowsapp.lib' -f $executablePath, $objectPath, $encoderObjectPath ) -join ' && ' & cmd.exe /d /s /c $compile diff --git a/native/probes/encoder-pool/CMakeLists.txt b/native/probes/encoder-pool/CMakeLists.txt new file mode 100644 index 0000000..9f7e9f8 --- /dev/null +++ b/native/probes/encoder-pool/CMakeLists.txt @@ -0,0 +1,27 @@ +cmake_minimum_required(VERSION 3.20) +project(screener_encoder_pool_probe LANGUAGES CXX) +set(CMAKE_CXX_STANDARD 20) +set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreaded) +set(WEBRTC_SDK "" CACHE PATH "Extracted matching Shiguredo WebRTC SDK") +set(MSVC_CRT_DIR "" CACHE PATH "Optional newer official MSVC x64 library directory") +if(NOT EXISTS "${WEBRTC_SDK}/lib/webrtc.lib") + message(FATAL_ERROR "This Windows probe needs WEBRTC_SDK pointing to the matching SDK") +endif() +add_executable(encoder-pool-probe harness.cpp h264_encoder_adapter.cpp + ../../capture/windows/h264_encoder.cpp) +target_include_directories(encoder-pool-probe SYSTEM PRIVATE + "${WEBRTC_SDK}/include" + "${WEBRTC_SDK}/include/third_party/abseil-cpp" + "${WEBRTC_SDK}/include/third_party/boringssl/src/include" + "${WEBRTC_SDK}/include/third_party/libyuv/include" + "${WEBRTC_SDK}/include/third_party/libvpx/source/libvpx") +target_compile_definitions(encoder-pool-probe PRIVATE + NDEBUG _ITERATOR_DEBUG_LEVEL=0 WEBRTC_WIN RTC_ENABLE_H265 NOMINMAX WIN32_LEAN_AND_MEAN _WIN32_WINNT=0x0A00) +target_compile_options(encoder-pool-probe PRIVATE /EHsc /GR /O2 /W3) +if(MSVC_CRT_DIR) + target_link_directories(encoder-pool-probe BEFORE PRIVATE "${MSVC_CRT_DIR}") +endif() +target_link_libraries(encoder-pool-probe PRIVATE + "${WEBRTC_SDK}/lib/webrtc.lib" + winmm ws2_32 Strmiids crypt32 dmoguids iphlpapi msdmo Secur32 wmcodecdspuuid + ole32 oleaut32 mfplat mf mfuuid d3d11 dxgi dxguid evr) diff --git a/native/probes/encoder-pool/README.md b/native/probes/encoder-pool/README.md new file mode 100644 index 0000000..18c8986 --- /dev/null +++ b/native/probes/encoder-pool/README.md @@ -0,0 +1,99 @@ +# Encoder Pool Feasibility Probe + +Explicit Windows experiment, not a product dependency or CI job. Two real +libwebrtc VideoStreamEncoders consume one synthetic 640x360 source through +independent stock VideoAdapters. It does not capture the desktop or open ports. +[Research and evidence](../../../docs/research/webrtc-encoder-pool.md) own conclusions. + +Use Shiguredo `m152.7977.0.2` Windows x64 SDK (WebRTC commit +`6f37672d358475cd17544121a12494da454d85fb`), matching its headers and static library: +[download](https://github.com/shiguredo-webrtc-build/webrtc-build/releases/download/m152.7977.0.2/webrtc.windows_x86_64.zip). +ZIP SHA-256: `3250ee091eb745b9885482d92db352e6bcd5426940021ff90a095747d84d2137`. +Keep the extracted SDK, official compiler dependencies and executable under +ignored `build/encoder-pool`; preserve the SDK's notices. Do not ship this SDK +as part of Screener based on this probe. + +With CMake and Visual Studio 2022 x64 tools available: + +```powershell +cmake -S native/probes/encoder-pool -B build/encoder-pool/probe -A x64 ` + -DWEBRTC_SDK="$PWD/build/encoder-pool/webrtc" +cmake --build build/encoder-pool/probe --config Release -j 2 +& ./build/encoder-pool/probe/Release/encoder-pool-probe.exe +& ./build/encoder-pool/probe/Release/encoder-pool-probe.exe --pooled +& ./build/encoder-pool/probe/Release/encoder-pool-probe.exe --pooled --group-adjuster +``` + +The tested older compiler was MSVC 14.42; linking requires the newer official +14.44 libraries/linker used by the SDK. On such a machine pass `MSVC_CRT_DIR` +to CMake and `/p:LinkToolPath=` to MSBuild after `--`. +Do not supply substitute STL implementations or alter global tool settings. + +The full trace takes about 51 seconds: healthy, 100 kbps weak B, release to +2 Mbps, one omitted B input, then independent B retirement. `--healthy-only` +omits the weak/release phases. JSONL reports real encoding calls, both WebRTC +decoders, raw libvpx hashes without history-dependent postprocessing, dimensions, +first transition times, group lifetime and resource observations. `integrity` +checks nonzero decoding, error counts, shared-payload comparisons and retirement; +it is not an automatic performance/parity verdict. + +`--no-adjuster` is a diagnostic ablation only. `--group-adjuster` instead keeps +WebRTC's actual adjustment algorithm once per physical codec; it must not run +alongside per-VSE correction. `--unsafe-skip` removes dependency protection as a +negative control and should fail integrity with group adjustment and a skipped +input. This cache arm is guarded single-source realtime VP8/L1T1 with synchronous +codecs; no H264 cache, physical overload or public-network claim follows. + +`--realtime` removes per-input encoder-queue drains. `--latency-probe` requires +that mode and inserts a bounded pixel-proportional 45 ms codec delay, then +releases it; it is not a CPU/GPU stress test. `--shared-pipeline --realtime +--latency-probe` is a separate control with one full stock VSE feeding both +decoders, not two VSEs borrowing its encoder. Do not combine it with pool or +adjuster flags. Source and codec work remain bounded; use a 95-second process +deadline for these approximately 70-second traces. The shared-pipeline B resource +metrics repeat the common pipeline, while its decoded counters are independent. + +## Hardware H264 Pipeline + +The separate `--h264` arm wraps the existing Windows MFT `LiveEncoder` in a +WebRTC encoder adapter. It requires `--shared-pipeline --healthy-only --export +PATH`; one complete stock VSE supplies two output callbacks. It does not test +two independent controllers borrowing an H264 encoder. Run it alone, with a +30-second process deadline and no concurrent capture/codec workload: + +```powershell +& ./build/encoder-pool/probe/Release/encoder-pool-probe.exe ` + --h264 --shared-pipeline --healthy-only --realtime ` + --export build/encoder-pool/h264-input.jsonl +``` + +This is synthetic I420 upload to NV12/MFT, not WGC capture or zero-copy input. +The pinned SDK has no H264 decoder, so this arm records payload equality and +encoded dimensions with decoded counters at zero. Validate the exported AUs +separately in Chrome. The existing `scripts/encoded-group-decode.mjs` accepts +`SCREENER_ENCODED_CODEC=avc1.42c033` and `SCREENER_ENCODED_OUTPUT` pointing to two +arrays of `{Index, Width, Height, PTS, Recovery, Data}` frames (nanosecond PTS, +base64 Annex-B data). A repeated export in those two arrays proves decoding of +that bitstream, not two Pion transports or weak-budget/resource adaptation. + +## Healthy Relay Chain + +Export the first 180 real AUs from sender A for the opt-in Go forwarding check: + +```powershell +& ./build/encoder-pool/probe/Release/encoder-pool-probe.exe ` + --pooled --group-adjuster --healthy-only --export build/encoder-pool/chain-input.jsonl +go test -c -o build/embedded-media/mediaedge.test.exe ./internal/client/mediaedge +$env:SCREENER_POOL_CHAIN_FIXTURE = "$PWD/build/encoder-pool/chain-input.jsonl" +$env:SCREENER_NATIVE_CAPTURE = "$PWD/build/client-check/screener-client-capture.exe" +$env:SCREENER_POOL_CHAIN_OUTPUT = "$PWD/build/encoder-pool/chain-result.json" +& ./build/embedded-media/mediaedge.test.exe ` + '-test.run=^TestRelayChainEncodedFixture$' '-test.v' '-test.timeout=30s' +``` + +The real capture executable must already be built, so Native relay capability +remains enabled while checking that healthy forwarding does not start it. This +replays encoded video through two Native receivers and three leaves, checking +recovery-first, contiguous, hash-identical AUs and closure. No frame payload is +committed. Optional output contains only the synthetic trace's summary. It is +not a live encoder/BWE integration or an internet/NAT test. diff --git a/native/probes/encoder-pool/h264_encoder_adapter.cpp b/native/probes/encoder-pool/h264_encoder_adapter.cpp new file mode 100644 index 0000000..f8d3921 --- /dev/null +++ b/native/probes/encoder-pool/h264_encoder_adapter.cpp @@ -0,0 +1,310 @@ +#include "h264_encoder_adapter.h" + +#include "../../capture/windows/h264_encoder.h" + +#include +#include +#include +#include +#include +#include + +#include "api/video_codecs/h264_profile_level_id.h" +#include "libyuv/convert_from.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "modules/video_coding/include/video_error_codes.h" +#include "system_wrappers/include/clock.h" + +namespace screener::probe { +namespace { +namespace native = screener::capture::windows; + +bool ValidDimensions(int width, int height) { + return width >= 2 && width <= 2560 && height >= 2 && height <= 1440 && + width % 2 == 0 && height % 2 == 0; +} + +webrtc::SdpVideoFormat HardwareFormat() { + return {"H264", {{"profile-level-id", "42c033"}, + {"level-asymmetry-allowed", "1"}, + {"packetization-mode", "1"}}, + {webrtc::ScalabilityMode::kL1T1}}; +} + +class HardwareEncoder final : public webrtc::VideoEncoder { + public: + HardwareEncoder(const webrtc::Environment& env, unsigned adapter_index, + unsigned encoder_index, webrtc::H264Level maximum_level) + : env_(env), adapter_index_(adapter_index), encoder_index_(encoder_index), + maximum_level_(maximum_level) {} + ~HardwareEncoder() override { Release(); } + + int InitEncode(const webrtc::VideoCodec* config, const Settings&) override { + Release(); + if (!config || config->codecType != webrtc::kVideoCodecH264 || + !ValidDimensions(config->width, config->height) || + config->maxFramerate == 0 || config->maxFramerate > 60 || + config->startBitrate == 0 || + config->startBitrate > std::numeric_limits::max() / 1000) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + if (config->numberOfSimulcastStreams > 1 || + config->H264().numberOfTemporalLayers > 1 || + (config->GetScalabilityMode() && + config->GetScalabilityMode() != webrtc::ScalabilityMode::kL1T1)) { + return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED; + } + profile_.width = config->width; + profile_.height = config->height; + profile_.frame_rate = config->maxFramerate; + profile_.bit_rate = config->startBitrate * 1000; + if (profile_.h264_level() > static_cast(maximum_level_)) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + try { + runtime_ = std::make_unique(); + auto adapters = native::EnumerateAdapters(); + const auto& adapter = native::SelectAdapter(adapters, adapter_index_); + device_ = native::CreateDevice(adapter); + auto activations = native::EnumerateHardwareEncoders(adapter); + encoder_ = std::make_unique( + native::ActivateTransform(activations, encoder_index_, device_.manager.Get()), + profile_); + nv12_.resize(static_cast(profile_.width) * profile_.height * 3 / 2); + implementation_name_ = encoder_->name + " (" + encoder_->identity + ")"; + applied_bitrate_ = profile_.bit_rate; + content_type_ = config->mode == webrtc::VideoCodecMode::kScreensharing + ? webrtc::VideoContentType::SCREENSHARE + : webrtc::VideoContentType::UNSPECIFIED; + return WEBRTC_VIDEO_CODEC_OK; + } catch (const std::exception& error) { + ReportFailure("initialize", error); + Release(); + return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; + } + } + + int32_t RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback* callback) override { + callback_ = callback; + return WEBRTC_VIDEO_CODEC_OK; + } + + int32_t Release() override { + encoder_.reset(); + device_ = {}; + runtime_.reset(); + nv12_.clear(); + applied_bitrate_ = 0; + paused_ = false; + key_frame_pending_ = true; + error_ = WEBRTC_VIDEO_CODEC_OK; + return WEBRTC_VIDEO_CODEC_OK; + } + + void SetRates(const RateControlParameters& rates) override { + if (!encoder_ || error_ != WEBRTC_VIDEO_CODEC_OK) return; + const UINT32 bitrate = rates.bitrate.get_sum_bps(); + if (bitrate != rates.bitrate.GetBitrate(0, 0) || + !std::isfinite(rates.framerate_fps)) { + error_ = WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + return; + } + if (bitrate == 0) { + paused_ = true; + return; + } + const double fps = rates.framerate_fps > 0 + ? rates.framerate_fps : profile_.frame_rate; + // Chromium M152 MF keeps the configured FPS and compensates its bitrate: + // media/gpu/windows/media_foundation_video_encode_accelerator_win.cc, + // AdjustBitrateToFrameRate (152.0.7977.82). + const double adjusted = static_cast(bitrate) * profile_.frame_rate / fps; + if (!std::isfinite(adjusted) || adjusted < 1 || + adjusted > std::numeric_limits::max()) { + error_ = WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + return; + } + const auto next_bitrate = static_cast(adjusted); + try { + if (next_bitrate != applied_bitrate_) encoder_->SetBitrate(next_bitrate); + applied_bitrate_ = next_bitrate; + key_frame_pending_ = key_frame_pending_ || paused_; + paused_ = false; + } catch (const std::exception& error) { + ReportFailure("rates", error); + error_ = WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; + } + } + + int32_t Encode(const webrtc::VideoFrame& frame, + const std::vector* requested) override { + if (!encoder_ || !callback_) return WEBRTC_VIDEO_CODEC_UNINITIALIZED; + if (error_ != WEBRTC_VIDEO_CODEC_OK) return error_; + key_frame_pending_ = key_frame_pending_ || + (requested && std::find(requested->begin(), requested->end(), + webrtc::VideoFrameType::kVideoFrameKey) != requested->end()); + if (paused_ || (requested && !requested->empty() && + requested->front() == webrtc::VideoFrameType::kEmptyFrame)) { + callback_->OnFrameDropped(frame.rtp_timestamp(), 0, true); + return WEBRTC_VIDEO_CODEC_OK; + } + if (!frame.video_frame_buffer() || frame.width() != static_cast(profile_.width) || + frame.height() != static_cast(profile_.height) || frame.timestamp_us() < 0 || + frame.timestamp_us() > std::numeric_limits::max() / 10) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + try { + const auto began = env_.clock().TimeInMilliseconds(); + auto input = frame.video_frame_buffer()->ToI420(); + if (!input || libyuv::I420ToNV12( + input->DataY(), input->StrideY(), input->DataU(), input->StrideU(), + input->DataV(), input->StrideV(), nv12_.data(), profile_.width, + nv12_.data() + static_cast(profile_.width) * profile_.height, + profile_.width, profile_.width, profile_.height) != 0) { + return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; + } + D3D11_TEXTURE2D_DESC description{}; + description.Width = profile_.width; + description.Height = profile_.height; + description.MipLevels = 1; + description.ArraySize = 1; + description.Format = DXGI_FORMAT_NV12; + description.SampleDesc.Count = 1; + description.Usage = D3D11_USAGE_DEFAULT; + D3D11_SUBRESOURCE_DATA pixels{}; + pixels.pSysMem = nv12_.data(); + pixels.SysMemPitch = profile_.width; + pixels.SysMemSlicePitch = static_cast(nv12_.size()); + native::ComPtr texture; + native::Check(device_.device->CreateTexture2D(&description, &pixels, &texture), + "webrtc-h264-upload"); + const auto access_unit = encoder_->Encode( + texture.Get(), static_cast(frame.timestamp_us()) * 10, + key_frame_pending_); + key_frame_pending_ = false; + webrtc::EncodedImage image; + image.SetEncodedData(webrtc::EncodedImageBuffer::Create( + access_unit.bytes.data(), access_unit.bytes.size())); + image._encodedWidth = profile_.width; + image._encodedHeight = profile_.height; + image.SetRtpTimestamp(frame.rtp_timestamp()); + image.capture_time_ms_ = frame.timestamp_us() / 1000; + image.ntp_time_ms_ = frame.ntp_time_ms(); + image.SetPresentationTimestamp(frame.presentation_timestamp()); + image.rotation_ = frame.rotation(); + image.content_type_ = content_type_; + image.SetColorSpace(frame.color_space()); + image.SetPacketInfos(frame.packet_infos()); + image.SetVideoFrameTrackingId(frame.id()); + image.SetSimulcastIndex(0); + image.SetSpatialIndex(0); + image.SetTemporalIndex(0); + image.set_end_of_temporal_unit(true); + image.SetFrameType(access_unit.key_frame + ? webrtc::VideoFrameType::kVideoFrameKey + : webrtc::VideoFrameType::kVideoFrameDelta); + image.SetEncodeTime(began, env_.clock().TimeInMilliseconds()); + webrtc::CodecSpecificInfo info; + info.codecType = webrtc::kVideoCodecH264; + info.codecSpecific.H264 = {}; + info.codecSpecific.H264.packetization_mode = webrtc::H264PacketizationMode::NonInterleaved; + info.codecSpecific.H264.idr_frame = access_unit.key_frame; + info.scalability_mode = webrtc::ScalabilityMode::kL1T1; + const auto result = callback_->OnEncodedImage(image, &info); + return result.error == webrtc::EncodedImageCallback::Result::OK + ? WEBRTC_VIDEO_CODEC_OK : WEBRTC_VIDEO_CODEC_ERROR; + } catch (const std::exception& error) { + ReportFailure("encode", error); + error_ = WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; + return error_; + } + } + + EncoderInfo GetEncoderInfo() const override { + EncoderInfo info; + info.implementation_name = implementation_name_; + info.is_hardware_accelerated = true; + info.enable_cpu_overuse_detection = true; + info.has_trusted_rate_controller = false; + info.is_qp_trusted = false; + info.scaling_settings = ScalingSettings::kOff; + info.requested_resolution_alignment = 2; + info.supports_native_handle = false; + info.supports_simulcast = false; + info.preferred_pixel_formats = {webrtc::VideoFrameBuffer::Type::kI420}; + return info; + } + + private: + static void ReportFailure(const char* operation, const std::exception& error) { + std::cerr << "hardware-h264 " << operation << ": " << error.what(); + if (const auto* failure = dynamic_cast(&error)) { + std::cerr << " stage=" << failure->stage() << " hresult=" << failure->result(); + } + std::cerr << '\n'; + } + + const webrtc::Environment env_; + const unsigned adapter_index_; + const unsigned encoder_index_; + const webrtc::H264Level maximum_level_; + std::unique_ptr runtime_; + native::DeviceContext device_; + std::unique_ptr encoder_; + native::VideoProfile profile_; + std::vector nv12_; + webrtc::EncodedImageCallback* callback_ = nullptr; + std::string implementation_name_ = "Windows Media Foundation H264"; + webrtc::VideoContentType content_type_ = webrtc::VideoContentType::UNSPECIFIED; + UINT32 applied_bitrate_ = 0; + int32_t error_ = WEBRTC_VIDEO_CODEC_OK; + bool paused_ = false; + bool key_frame_pending_ = true; +}; + +class HardwareFactory final : public webrtc::VideoEncoderFactory { + public: + HardwareFactory(unsigned adapter_index, unsigned encoder_index) + : adapter_index_(adapter_index), encoder_index_(encoder_index) {} + + std::vector GetSupportedFormats() const override { + return {HardwareFormat()}; + } + + CodecSupport QueryCodecSupport(const webrtc::SdpVideoFormat& format, + std::optional scalability_mode, + std::optional resolution) const override { + if (resolution && !ValidDimensions(resolution->width, resolution->height)) return {}; + const auto profile = webrtc::ParseSdpForH264ProfileLevelId(format.parameters); + native::VideoProfile minimum; + minimum.frame_rate = 1; + if (resolution) { + minimum.width = resolution->width; + minimum.height = resolution->height; + } + const bool supported = format.IsSameCodec(HardwareFormat()) && + (!scalability_mode || *scalability_mode == "L1T1") && profile && + minimum.h264_level() <= static_cast(profile->level); + return {supported, supported}; + } + + std::unique_ptr Create( + const webrtc::Environment& env, const webrtc::SdpVideoFormat& format) override { + if (!QueryCodecSupport(format, std::nullopt, std::nullopt).is_supported) return nullptr; + const auto profile = webrtc::ParseSdpForH264ProfileLevelId(format.parameters); + return std::make_unique(env, adapter_index_, encoder_index_, profile->level); + } + + private: + const unsigned adapter_index_; + const unsigned encoder_index_; +}; + +} // namespace + +std::unique_ptr CreateHardwareEncoderFactory( + unsigned adapter_index, unsigned encoder_index) { + return std::make_unique(adapter_index, encoder_index); +} + +} // namespace screener::probe diff --git a/native/probes/encoder-pool/h264_encoder_adapter.h b/native/probes/encoder-pool/h264_encoder_adapter.h new file mode 100644 index 0000000..3a3f883 --- /dev/null +++ b/native/probes/encoder-pool/h264_encoder_adapter.h @@ -0,0 +1,13 @@ +#pragma once + +#include + +#include "api/video_codecs/video_encoder_factory.h" + +namespace screener::probe { + +// CPU I420 input only. InitEncode and Release own COM/MF on the encoder queue. +std::unique_ptr CreateHardwareEncoderFactory( + unsigned adapter_index = 0, unsigned encoder_index = 0); + +} // namespace screener::probe diff --git a/native/probes/encoder-pool/harness.cpp b/native/probes/encoder-pool/harness.cpp new file mode 100644 index 0000000..469f11a --- /dev/null +++ b/native/probes/encoder-pool/harness.cpp @@ -0,0 +1,567 @@ +// Explicit synthetic experiment. Not part of Client, Server, unit tests or CI. +#include "pool.h" +#include "h264_encoder_adapter.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "api/environment/environment_factory.h" +#include "api/field_trials_view.h" +#include "api/make_ref_counted.h" +#include "api/task_queue/task_queue_base.h" +#include "api/task_queue/task_queue_factory.h" +#include "api/video/builtin_video_bitrate_allocator_factory.h" +#include "api/video/i420_buffer.h" +#include "api/video/video_adapter.h" +#include "api/video/video_frame.h" +#include "api/video/video_stream_encoder_settings.h" +#include "api/video_codecs/builtin_video_decoder_factory.h" +#include "api/video_codecs/video_decoder.h" +#include "api/video_codecs/video_decoder_factory.h" +#include "call/video_send_stream.h" +#include "system_wrappers/include/clock.h" +#include "video/adaptation/overuse_frame_detector.h" +#include "video/frame_cadence_adapter.h" +#include "video/send_statistics_proxy.h" +#include "video/video_stream_encoder.h" +#include "vpx/vpx_decoder.h" +#include "vpx/vp8dx.h" + +namespace { +using namespace webrtc; +constexpr int kWidth = 640; +constexpr int kHeight = 360; +constexpr int kFps = 30; + +class Trials final : public FieldTrialsView { + public: + explicit Trials(bool disable) : disable_(disable) {} + std::string Lookup(absl::string_view key) const override { + return disable_ && key == "WebRTC-VideoRateControl" ? "bitrate_adjuster:false" : ""; + } + private: + bool disable_; +}; + +template +void Sync(TaskQueueBase* queue, F fn) { + auto completion = std::make_shared>(); + auto future = completion->get_future(); + queue->PostTask([completion, fn = std::move(fn)]() mutable { + try { + fn(); + completion->set_value(); + } catch (...) { + completion->set_exception(std::current_exception()); + } + }); + if (future.wait_for(std::chrono::seconds(10)) != std::future_status::ready) + throw std::runtime_error("task queue deadline exceeded"); + future.get(); +} + +uint64_t HashBytes(uint64_t hash, const uint8_t* bytes, size_t count) { + for (size_t i = 0; i < count; ++i) hash = (hash ^ bytes[i]) * 1099511628211ULL; + return hash; +} + +struct FrameProof { + uint64_t payload = 0; + uint64_t pixels = 0; + int width = 0; + int height = 0; +}; + +class SharedProof { + public: + void Observe(int consumer, uint32_t timestamp, FrameProof proof) { + std::lock_guard lock(mutex_); + auto& row = pending_[timestamp]; + row[consumer] = proof; + if (row[0].payload && row[1].payload) { + if (row[0].payload == row[1].payload) { + ++compared; + if (row[0].pixels && row[1].pixels) { + ++decodedCompared; + if (row[0].pixels != row[1].pixels || row[0].width != row[1].width || + row[0].height != row[1].height) ++mismatches; + } + } + pending_.erase(timestamp); + } + while (pending_.size() > 64) pending_.erase(pending_.begin()); + } + std::atomic compared{0}; + std::atomic decodedCompared{0}; + std::atomic mismatches{0}; + + private: + std::mutex mutex_; + std::map> pending_; +}; + +struct Counts { + int encoded = 0; + int decoded = 0; + int errors = 0; + int keyframes = 0; + int width = 0; + int height = 0; + int min_width = kWidth; + int min_height = kHeight; + uint64_t bytes = 0; + double first_full_ms = -1; + double first_reduced_ms = -1; +}; + +class Sink final : public VideoStreamEncoderInterface::EncoderSink, + public DecodedImageCallback { + public: + Sink(const Environment& env, int consumer, SharedProof& proof, bool h264 = false) + : consumer_(consumer), proof_(proof) { + if (h264) return; // The pinned SDK has no H264 decoder; export to Browser WebCodecs. + auto factory = CreateBuiltinVideoDecoderFactory(); + decoder_ = factory->Create(env, SdpVideoFormat("VP8")); + VideoDecoder::Settings settings; + settings.set_codec_type(kVideoCodecVP8); + settings.set_number_of_cores(1); + if (!decoder_ || !decoder_->Configure(settings) || + decoder_->RegisterDecodeCompleteCallback(this) != 0) + throw std::runtime_error("VP8 decoder initialization failed"); + vpx_codec_dec_cfg_t raw_config{1, 0, 0}; + if (vpx_codec_dec_init(&raw_, vpx_codec_vp8_dx(), &raw_config, 0) != VPX_CODEC_OK) + throw std::runtime_error("raw VP8 decoder initialization failed"); + } + + ~Sink() override { + if (decoder_) { decoder_->Release(); vpx_codec_destroy(&raw_); } + } + + void OnEncoderConfigurationChanged(std::vector, bool, + VideoEncoderConfig::ContentType, int) override {} + void OnBitrateAllocationUpdated(const VideoBitrateAllocation&) override {} + void OnVideoLayersAllocationUpdated(VideoLayersAllocation) override {} + void OnFrameDropped(uint32_t, int, bool) override {} + + Result OnEncodedImage(const EncodedImage& image, + const CodecSpecificInfo*) override { + std::lock_guard lock(mutex_); + ++counts_.encoded; + counts_.bytes += image.size(); + counts_.keyframes += image.IsKey(); + if (export_.is_open() && counts_.encoded <= 180) { + constexpr char hex[] = "0123456789abcdef"; + std::string data; + data.reserve(image.size() * 2); + for (uint8_t byte : image) { + data.push_back(hex[byte >> 4]); + data.push_back(hex[byte & 15]); + } + export_ << "{\"index\":" << counts_.encoded - 1 + << ",\"width\":" << image._encodedWidth + << ",\"height\":" << image._encodedHeight + << ",\"recovery\":" << (image.IsKey() ? "true" : "false") + << ",\"dataHex\":\"" << data << "\"}\n"; + if (!export_) ++counts_.errors; + } + current_timestamp_ = image.RtpTimestamp(); + current_payload_ = HashBytes(1469598103934665603ULL, image.data(), image.size()); + if (!decoder_) { + RecordSize(image._encodedWidth, image._encodedHeight); + proof_.Observe(consumer_, current_timestamp_, + {current_payload_, 0, static_cast(image._encodedWidth), static_cast(image._encodedHeight)}); + } else if (decoder_->Decode(image, 0) < 0) ++counts_.errors; + if (decoder_ && vpx_codec_decode(&raw_, image.data(), static_cast(image.size()), nullptr, 0) != VPX_CODEC_OK) { + ++counts_.errors; + } else if (decoder_) { + vpx_codec_iter_t iterator = nullptr; + if (const auto* raw = vpx_codec_get_frame(&raw_, &iterator)) { + uint64_t hash = 1469598103934665603ULL; + for (int plane = 0; plane < 3; ++plane) { + const unsigned width = plane ? (raw->d_w + 1) / 2 : raw->d_w; + const unsigned height = plane ? (raw->d_h + 1) / 2 : raw->d_h; + for (unsigned y = 0; y < height; ++y) + hash = HashBytes(hash, raw->planes[plane] + y * raw->stride[plane], width); + } + proof_.Observe(consumer_, current_timestamp_, + {current_payload_, hash, static_cast(raw->d_w), static_cast(raw->d_h)}); + } else ++counts_.errors; + } + if (duplicate_) duplicate_->OnEncodedImage(image, nullptr); + return Result(Result::OK); + } + + // The selected built-in VP8 decoder completes synchronously inside Decode. + int32_t Decoded(VideoFrame& frame) override { + ++counts_.decoded; + RecordSize(frame.width(), frame.height()); + return 0; + } + + void RecordSize(int width, int height) { + counts_.width = width; + counts_.height = height; + counts_.min_width = std::min(counts_.min_width, width); + counts_.min_height = std::min(counts_.min_height, height); + const double elapsed = std::chrono::duration( + std::chrono::steady_clock::now() - phase_started_).count(); + if (width == kWidth && height == kHeight && counts_.first_full_ms < 0) + counts_.first_full_ms = elapsed; + if (width < kWidth && counts_.first_reduced_ms < 0) + counts_.first_reduced_ms = elapsed; + } + + Counts Snapshot() { + std::lock_guard lock(mutex_); + return counts_; + } + void Export(const std::string& path) { + export_.open(path, std::ios::binary | std::ios::trunc); + if (!export_) throw std::runtime_error("fixture export unavailable"); + } + void SetDuplicate(Sink* sink) { + std::lock_guard lock(mutex_); + duplicate_ = sink; + } + void FinishExport() { + if (!export_.is_open()) return; + export_.flush(); + export_.close(); + if (!export_ || counts_.encoded < 180) + throw std::runtime_error("fixture export incomplete"); + } + void ResetMinimum() { + std::lock_guard lock(mutex_); + counts_.min_width = kWidth; + counts_.min_height = kHeight; + counts_.first_full_ms = -1; + counts_.first_reduced_ms = -1; + phase_started_ = std::chrono::steady_clock::now(); + } + + private: + const int consumer_; + SharedProof& proof_; + std::mutex mutex_; + std::unique_ptr decoder_; + vpx_codec_ctx_t raw_{}; + std::ofstream export_; + Sink* duplicate_ = nullptr; + Counts counts_; + uint32_t current_timestamp_ = 0; + uint64_t current_payload_ = 0; + std::chrono::steady_clock::time_point phase_started_ = std::chrono::steady_clock::now(); +}; + +// Each sender applies stock VideoAdapter wants to the same original input. +class Source final : public VideoSourceInterface { + public: + void AddOrUpdateSink(VideoSinkInterface* sink, + const VideoSinkWants& wants) override { + sink_ = sink; + adapter_.OnSinkWants(wants); + ++wants_updates; + min_pixels = std::min(min_pixels, wants.max_pixel_count); + } + void RemoveSink(VideoSinkInterface* sink) override { + if (sink_ == sink) sink_ = nullptr; + } + void Push(const VideoFrame& original) { + if (!sink_) return; + int crop_width, crop_height, width, height; + if (!adapter_.AdaptFrameResolution(original.width(), original.height(), + original.timestamp_us() * 1000, + &crop_width, &crop_height, &width, &height)) { + ++dropped; + sink_->OnDiscardedFrame(); + return; + } + VideoFrame frame = original; + if (width != original.width() || height != original.height() || + crop_width != original.width() || crop_height != original.height()) { + auto scaled = I420Buffer::Create(width, height); + scaled->CropAndScaleFrom(*original.video_frame_buffer()->ToI420(), + (original.width() - crop_width) / 2, + (original.height() - crop_height) / 2, + crop_width, crop_height); + frame.set_video_frame_buffer(scaled); + } + sink_->OnFrame(frame); + } + int dropped = 0; + int wants_updates = 0; + int min_pixels = kWidth * kHeight; + + private: + VideoSinkInterface* sink_ = nullptr; + VideoAdapter adapter_{2}; +}; + +struct Pipeline { + Source source; + std::unique_ptr stats; + std::unique_ptr sink; + std::unique_ptr encoder; + TaskQueueBase* encode_queue = nullptr; +}; + +VideoEncoderConfig Config(const SdpVideoFormat& format) { + VideoEncoderConfig config; + config.codec_type = format.name == "H264" ? kVideoCodecH264 : kVideoCodecVP8; + config.video_format = format; + config.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo; + config.number_of_streams = 1; + config.max_bitrate_bps = 2'000'000; + config.frame_drop_enabled = true; + config.is_quality_scaling_allowed = true; + config.simulcast_layers.resize(1); + config.simulcast_layers[0].active = true; + config.simulcast_layers[0].max_framerate = kFps; + config.simulcast_layers[0].max_bitrate_bps = 2'000'000; + config.simulcast_layers[0].num_temporal_layers = 1; + if (config.codec_type == kVideoCodecVP8) { + auto vp8 = VideoEncoder::GetDefaultVp8Settings(); + vp8.automaticResizeOn = true; + config.encoder_specific_settings = + make_ref_counted(vp8); + } + return config; +} + +void Start(Pipeline& pipeline, const Environment& env, PoolFactory& factory, + VideoBitrateAllocatorFactory* allocator, int consumer, SharedProof& proof) { + VideoSendStream::Config send_config(nullptr); + const auto format = factory.GetSupportedFormats().front(); + send_config.rtp.ssrcs = {static_cast(100 + consumer)}; + send_config.rtp.payload_name = format.name; + send_config.rtp.payload_type = 96; + pipeline.stats = std::make_unique( + &env.clock(), send_config, VideoEncoderConfig::ContentType::kRealtimeVideo, + env.field_trials()); + pipeline.sink = std::make_unique(env, consumer, proof, format.name == "H264"); + VideoStreamEncoderSettings settings(VideoEncoder::Capabilities(false)); + settings.encoder_factory = &factory; + settings.bitrate_allocator_factory = allocator; + auto queue = env.task_queue_factory().CreateTaskQueue( + "PoolProbeEncoder", TaskQueueFactory::Priority::kNormal); + pipeline.encode_queue = queue.get(); + auto cadence = FrameCadenceAdapterInterface::Create( + &env.clock(), queue.get(), nullptr, TaskQueueBase::Current(), env.field_trials()); + pipeline.encoder = std::make_unique( + env, 1, pipeline.stats.get(), settings, + std::make_unique(env, pipeline.stats.get()), + std::move(cadence), std::move(queue), + VideoStreamEncoder::BitrateAllocationCallbackType::kVideoBitrateAllocation); + pipeline.encoder->SetSink(pipeline.sink.get(), false); + pipeline.encoder->SetSource(&pipeline.source, DegradationPreference::MAINTAIN_FRAMERATE); + pipeline.encoder->SetStartBitrate(2'000'000); + pipeline.encoder->ConfigureEncoder(Config(format), 1200); + pipeline.encoder->OnBitrateUpdated(DataRate::BitsPerSec(2'000'000), + DataRate::BitsPerSec(2'000'000), 0, 20, 0); +} + +VideoFrame MakeFrame(const Environment& env, int index) { + auto pixels = I420Buffer::Create(kWidth, kHeight); + for (int y = 0; y < kHeight; ++y) { + for (int x = 0; x < kWidth; ++x) { + const int shifted = x + index * 5; + pixels->MutableDataY()[y * pixels->StrideY() + x] = + static_cast(32 + ((shifted * 3 + y * 2 + + (((shifted / 23) ^ (y / 17)) & 7) * 19) % 192)); + } + } + for (int y = 0; y < kHeight / 2; ++y) { + for (int x = 0; x < kWidth / 2; ++x) { + pixels->MutableDataU()[y * pixels->StrideU() + x] = + static_cast(80 + (x + index * 2) % 96); + pixels->MutableDataV()[y * pixels->StrideV() + x] = + static_cast(80 + (y + index) % 96); + } + } + return VideoFrame::Builder().set_video_frame_buffer(pixels) + .set_timestamp_us(env.clock().TimeInMicroseconds()) + .set_ntp_time_ms(env.clock().CurrentNtpInMilliseconds()) + .set_id(static_cast(index)).build(); +} + +void PrintCounts(const Counts& before, const Counts& after, double seconds) { + std::cout << "{\"encoded\":" << after.encoded - before.encoded + << ",\"decoded\":" << after.decoded - before.decoded + << ",\"fps\":" << (after.decoded - before.decoded) / seconds + << ",\"kbps\":" << (after.bytes - before.bytes) * 0.008 / seconds + << ",\"errors\":" << after.errors - before.errors + << ",\"keyframes\":" << after.keyframes - before.keyframes + << ",\"width\":" << after.width << ",\"height\":" << after.height + << ",\"minWidth\":" << after.min_width + << ",\"minHeight\":" << after.min_height + << ",\"firstFullMs\":" << after.first_full_ms + << ",\"firstReducedMs\":" << after.first_reduced_ms << '}'; +} +} // namespace + +int main(int argc, char** argv) { + try { + const std::vector args(argv + 1, argv + argc); + const auto has = [&](const char* flag) { return std::find(args.begin(), args.end(), flag) != args.end(); }; + const bool pooled = has("--pooled"); + const bool group_adjuster = has("--group-adjuster"); + const bool no_adjuster = has("--no-adjuster") || group_adjuster; + const bool healthy_only = has("--healthy-only"); + const bool realtime = has("--realtime"); + const bool latency_probe = has("--latency-probe"); + if (latency_probe && !realtime) + throw std::runtime_error("--latency-probe requires --realtime"); + const bool shared_pipeline = has("--shared-pipeline"); + const bool h264 = has("--h264"); + if (h264 && (!shared_pipeline || !healthy_only || !has("--export"))) + throw std::runtime_error("--h264 requires --shared-pipeline --healthy-only --export PATH"); + if (shared_pipeline && (pooled || group_adjuster || no_adjuster || has("--unsafe-skip") || + (!healthy_only && !latency_probe))) + throw std::runtime_error("shared pipeline control requires a healthy/latency trace with stock VSE settings"); + const auto env = CreateEnvironment(std::make_unique(no_adjuster)); + const bool unsafe_skip = has("--unsafe-skip"); + PoolFactory factory(pooled, group_adjuster, unsafe_skip, + h264 ? screener::probe::CreateHardwareEncoderFactory() : nullptr); + auto allocator = CreateBuiltinVideoBitrateAllocatorFactory(); + auto worker = env.task_queue_factory().CreateTaskQueue( + "PoolProbeWorker", TaskQueueFactory::Priority::kNormal); + SharedProof proof; + std::array pipelines; + Sync(worker.get(), [&] { + Start(pipelines[0], env, factory, allocator.get(), 0, proof); + if (shared_pipeline) { + pipelines[1].sink = std::make_unique(env, 1, proof, h264); + pipelines[0].sink->SetDuplicate(pipelines[1].sink.get()); + } else Start(pipelines[1], env, factory, allocator.get(), 1, proof); + }); + const auto export_arg = std::find(args.begin(), args.end(), "--export"); + if (export_arg != args.end()) { + if (export_arg + 1 == args.end()) throw std::runtime_error("--export needs a path"); + pipelines[0].sink->Export(*(export_arg + 1)); + } + int frame_index = 0; + auto run = [&](const char* name, int seconds, int weak_bitrate, bool skip, bool retire) { + Sync(worker.get(), [&] { + if (retire) { + if (shared_pipeline) pipelines[0].sink->SetDuplicate(nullptr); + else { + pipelines[1].encoder->Stop(); + pipelines[1].encoder.reset(); + } + } + }); + const auto a = pipelines[0].sink->Snapshot(); + const auto b = pipelines[1].sink->Snapshot(); + pipelines[0].sink->ResetMinimum(); + pipelines[1].sink->ResetMinimum(); + const int encodes = factory.encodes.load(), hits = factory.hits.load(); + const auto started = std::chrono::steady_clock::now(); + for (int frame = 0; frame < seconds * kFps; ++frame) { + // Equal feedback cadence for both controllers; only B's budget differs. + if (frame % 6 == 0) Sync(worker.get(), [&] { + for (int i = 0; i < 2; ++i) { + if (!pipelines[i].encoder) continue; + const auto rate = DataRate::BitsPerSec(i == 0 ? 2'000'000 : weak_bitrate); + pipelines[i].encoder->OnBitrateUpdated(rate, rate, 0, 20, 0); + } + }); + auto original = MakeFrame(env, ++frame_index); + for (int consumer = 0; consumer < 2; ++consumer) { + if (!pipelines[consumer].encoder) continue; + if (consumer == 1 && (retire || (skip && frame == kFps))) continue; + Sync(worker.get(), [&, consumer] { pipelines[consumer].source.Push(original); }); + if (!realtime) Sync(pipelines[consumer].encode_queue, [] {}); + } + std::this_thread::sleep_until(started + + std::chrono::microseconds((frame + 1) * 1'000'000LL / kFps)); + } + for (auto& pipeline : pipelines) { + if (pipeline.encoder) Sync(pipeline.encode_queue, [] {}); + } + Sync(worker.get(), [] {}); + const double elapsed = std::chrono::duration( + std::chrono::steady_clock::now() - started).count(); + std::cout << "{\"phase\":\"" << name << "\",\"seconds\":" << elapsed + << ",\"encodes\":" << factory.encodes.load() - encodes + << ",\"hits\":" << factory.hits.load() - hits << ",\"a\":"; + PrintCounts(a, pipelines[0].sink->Snapshot(), elapsed); + std::cout << ",\"b\":"; + PrintCounts(b, pipelines[1].sink->Snapshot(), elapsed); + const auto stats_a = pipelines[0].stats->GetStats(); + const auto stats_b = shared_pipeline ? stats_a : pipelines[1].stats->GetStats(); + std::cout << ",\"aEncodeMs\":" << stats_a.avg_encode_time_ms + << ",\"bEncodeMs\":" << stats_b.avg_encode_time_ms + << ",\"aEncodeUsage\":" << stats_a.encode_usage_percent + << ",\"bEncodeUsage\":" << stats_b.encode_usage_percent + << ",\"aCpuLimitedResolution\":" << (stats_a.cpu_limited_resolution ? "true" : "false") + << ",\"bCpuLimitedResolution\":" << (stats_b.cpu_limited_resolution ? "true" : "false") + << ",\"aCpuLimitedFramerate\":" << (stats_a.cpu_limited_framerate ? "true" : "false") + << ",\"bCpuLimitedFramerate\":" << (stats_b.cpu_limited_framerate ? "true" : "false") + << "}\n" << std::flush; + }; + run("healthy", 6, 2'000'000, false, false); + if (latency_probe) { + factory.simulatedLatencyMs = 45; + run("encoder-delayed", 30, 2'000'000, false, false); + factory.simulatedLatencyMs = 0; + run("encoder-released", 30, 2'000'000, false, false); + } else if (!healthy_only) { + run("weak", 15, 100'000, false, false); + run("released", 25, 2'000'000, false, false); + } + if (!shared_pipeline) run("skipped-input", 3, 2'000'000, true, false); + run("retired-b", 2, 2'000'000, false, true); + Sync(worker.get(), [&] { + pipelines[0].encoder->Stop(); + pipelines[0].encoder.reset(); + }); + pipelines[0].sink->FinishExport(); + const auto a = pipelines[0].sink->Snapshot(), b = pipelines[1].sink->Snapshot(); + const bool integrity = (h264 ? a.encoded > 0 && b.encoded > 0 : a.decoded > 0 && b.decoded > 0) && factory.live.load() == 0 && + a.errors + b.errors + proof.mismatches.load() == 0 && + (!pooled || factory.hits.load() > 0 && proof.compared.load() > 0) && + (!shared_pipeline || proof.compared.load() > 0); + std::cout << "{\"integrity\":\"" << (integrity ? "pass" : "fail") + << "\",\"codec\":\"" << (h264 ? "h264" : "vp8") + << "\",\"pooled\":" << (pooled ? "true" : "false") + << ",\"adjuster\":" << (no_adjuster ? "false" : "true") + << ",\"groupAdjuster\":" << (group_adjuster ? "true" : "false") + << ",\"unsafeSkip\":" << (unsafe_skip ? "true" : "false") + << ",\"realtime\":" << (realtime ? "true" : "false") + << ",\"latencyProbe\":" << (latency_probe ? "true" : "false") + << ",\"sharedPipeline\":" << (shared_pipeline ? "true" : "false") + << ",\"encodes\":" << factory.encodes.load() + << ",\"hits\":" << factory.hits.load() + << ",\"splits\":" << factory.splits.load() + << ",\"rateSplits\":" << factory.rateSplits.load() + << ",\"dependencySplits\":" << factory.dependencySplits.load() + << ",\"created\":" << factory.created.load() + << ",\"joins\":" << factory.joins.load() + << ",\"live\":" << factory.live.load() + << ",\"maxLive\":" << factory.maxLive.load() + << ",\"callbacks\":" << factory.callbacks.load() + << ",\"samePayloadCompared\":" << proof.compared.load() + << ",\"decodedHashComparisons\":" << proof.decodedCompared.load() + << ",\"decodedHashMismatches\":" << proof.mismatches.load() + << ",\"aWantsUpdates\":" << pipelines[0].source.wants_updates + << ",\"bWantsUpdates\":" << pipelines[1].source.wants_updates + << ",\"aMinPixels\":" << pipelines[0].source.min_pixels + << ",\"bMinPixels\":" << pipelines[1].source.min_pixels << "}\n"; + return integrity ? 0 : 1; + } catch (const std::exception& error) { + std::cerr << "pool harness failed: " << error.what() << '\n'; + return 2; + } +} diff --git a/native/probes/encoder-pool/pool.h b/native/probes/encoder-pool/pool.h new file mode 100644 index 0000000..861279d --- /dev/null +++ b/native/probes/encoder-pool/pool.h @@ -0,0 +1,316 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "api/environment/environment.h" +#include "api/video_codecs/builtin_video_encoder_factory.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "rtc_base/checks.h" +#include "video/encoder_bitrate_adjuster.h" + +// A one-source, single-stream experiment, not a product encoder registry. +// H264 uses the shared-pipeline control, not per-sender cache adaptation. +// Matching requests share only one last completed access unit. No warm cache. +// ponytail: one mutex serializes groups; validate per-group workers before production. +class PoolFactory final : public webrtc::VideoEncoderFactory { + public: + explicit PoolFactory(bool pooled, bool group_adjuster = false, bool unsafe_skip = false, + std::unique_ptr backend = nullptr) + : pooled_(pooled), group_adjuster_(group_adjuster), unsafe_skip_(unsafe_skip), + builtin_(backend ? std::move(backend) : webrtc::CreateBuiltinVideoEncoderFactory()), + format_(builtin_->GetSupportedFormats().front()) {} + std::atomic encodes{0}, hits{0}, splits{0}, live{0}, maxLive{0}, callbacks{0}; + std::atomic dependencySplits{0}, rateSplits{0}; + std::atomic created{0}, joins{0}; + std::atomic simulatedLatencyMs{0}; + + std::vector GetSupportedFormats() const override { + return {format_}; + } + + private: + struct Group final : webrtc::EncodedImageCallback { + PoolFactory& owner; + webrtc::VideoCodec config; + webrtc::VideoEncoder::Settings settings; + std::unique_ptr codec; + std::optional rates; + std::optional applied_rates; + std::unique_ptr adjuster; + float loss = 0; + int64_t rtt = 0; + std::optional timestamp, previous, attempted; + std::optional image; + webrtc::CodecSpecificInfo info; + bool dropped = false; + bool pendingKey = false; + + Group(PoolFactory& parent, const webrtc::Environment& env, + const webrtc::VideoCodec& configuration, + const webrtc::VideoEncoder::Settings& options) + : owner(parent), config(configuration), settings(options), + codec(parent.builtin_->Create(env, parent.format_)) { + RTC_CHECK(codec); + RTC_CHECK_EQ(codec->InitEncode(&config, settings), 0); + codec->RegisterEncodeCompleteCallback(this); + if (owner.group_adjuster_) { + adjuster = std::make_unique(config, env.field_trials(), env.clock()); + adjuster->OnEncoderInfo(codec->GetEncoderInfo()); + } + ++owner.created; + const int count = ++owner.live; + owner.maxLive.store(std::max(owner.maxLive.load(), count)); + } + ~Group() override { + codec->RegisterEncodeCompleteCallback(nullptr); + codec->Release(); + --owner.live; + } + Result OnEncodedImage(const webrtc::EncodedImage& output, + const webrtc::CodecSpecificInfo* details) override { + RTC_CHECK(!image.has_value()); + image = output; + if (details) info = *details; + if (adjuster) adjuster->OnEncodedFrame(webrtc::DataSize::Bytes(output.size()), 0, 0); + return Result(Result::OK); + } + void OnFrameDropped(uint32_t, int, bool) override { + dropped = true; + } + void UpdateCodecRates() { + if (!rates) return; + auto adjusted = *rates; + if (adjuster) adjusted.bitrate = adjuster->AdjustRateAllocation(*rates); + if (applied_rates != adjusted) codec->SetRates(adjusted); + applied_rates = adjusted; + } + }; + + static bool SameConfig(const Group& group, const webrtc::VideoCodec& c, + const webrtc::VideoEncoder::Settings& s) { + const auto& a = group.config; + return a.width == c.width && a.height == c.height && + a.startBitrate == c.startBitrate && a.maxBitrate == c.maxBitrate && + a.minBitrate == c.minBitrate && a.maxFramerate == c.maxFramerate && + a.qpMax == c.qpMax && a.mode == c.mode && a.active == c.active && + a.GetFrameDropEnabled() == c.GetFrameDropEnabled() && + a.GetVideoEncoderComplexity() == c.GetVideoEncoderComplexity() && + a.GetScalabilityMode() == c.GetScalabilityMode() && a.codecType == c.codecType && + (a.codecType == webrtc::kVideoCodecVP8 ? a.VP8() == c.VP8() : a.H264() == c.H264()) && + group.settings.number_of_cores == s.number_of_cores && + group.settings.max_payload_size == s.max_payload_size && + group.settings.encoder_thread_limit == s.encoder_thread_limit; + } + + class Encoder final : public webrtc::VideoEncoder { + public: + Encoder(PoolFactory& owner, const webrtc::Environment& env) + : owner_(owner), env_(env), + metadata_(owner.builtin_->Create(env, owner.format_)) {} + ~Encoder() override { Release(); } + int InitEncode(const webrtc::VideoCodec* c, const Settings& s) override { + RTC_CHECK(c->codecType == webrtc::kVideoCodecVP8 || c->codecType == webrtc::kVideoCodecH264); + RTC_CHECK_LE(c->numberOfSimulcastStreams, 1); + RTC_CHECK_EQ(c->codecType == webrtc::kVideoCodecVP8 ? c->VP8().numberOfTemporalLayers : c->H264().numberOfTemporalLayers, 1); + RTC_CHECK_EQ(c->mode, webrtc::VideoCodecMode::kRealtimeVideo); + std::lock_guard lock(owner_.mutex_); + group_.reset(); + last_.reset(); + config_ = *c; + settings_ = s; + rates_.reset(); + group_ = FindOrCreate(false, 0); + return 0; + } + int32_t RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback* callback) override { + callback_ = callback; + return 0; + } + int32_t Release() override { + std::lock_guard lock(owner_.mutex_); + group_.reset(); + last_.reset(); + return 0; + } + void SetRates(const RateControlParameters& rates) override { rates_ = rates; } + void OnPacketLossRateUpdate(float loss) override { loss_ = loss; } + void OnRttUpdate(int64_t rtt) override { rtt_ = rtt; } + EncoderInfo GetEncoderInfo() const override { + std::lock_guard lock(owner_.mutex_); + return group_ ? group_->codec->GetEncoderInfo() : metadata_->GetEncoderInfo(); + } + int32_t Encode(const webrtc::VideoFrame& frame, + const std::vector* requested) override { + std::optional output; + webrtc::CodecSpecificInfo info; + bool dropped = false; + int result = 0; + { + std::lock_guard lock(owner_.mutex_); + const uint32_t ts = frame.rtp_timestamp(); + bool key = requested && std::find(requested->begin(), requested->end(), + webrtc::VideoFrameType::kVideoFrameKey) != requested->end(); + const auto before = group_; + if (!Compatible(*group_)) { + // Single-owner rate changes must retain the encoder's adaptation history. + if (group_.use_count() == 2) { + ApplyRates(*group_); + } else { + if (owner_.rateSplits.load() < 8 && group_->rates && rates_) { + std::cerr << "rate split " << group_->rates->bitrate.get_sum_bps() + << '/' << rates_->bitrate.get_sum_bps() << " fps " + << group_->rates->framerate_fps << '/' << rates_->framerate_fps + << " target " << group_->rates->target_bitrate.get_sum_bps() + << '/' << rates_->target_bitrate.get_sum_bps() + << " allocation " << group_->rates->bandwidth_allocation.bps() + << '/' << rates_->bandwidth_allocation.bps() + << " rtt " << group_->rtt << '/' << rtt_ + << " loss " << group_->loss << '/' << loss_ << '\n'; + } + group_.reset(); + group_ = FindOrCreate(true, ts); + ++owner_.rateSplits; + } + } + if (owner_.pooled_) { + for (auto& reference : owner_.groups_) { + const auto candidate = reference.lock(); + if (candidate && candidate != group_ && Compatible(*candidate)) { + if (candidate->attempted == ts && candidate->image && + candidate->image->_frameType == webrtc::VideoFrameType::kVideoFrameKey) { + group_ = candidate; + ++owner_.joins; + break; + } + // Demand has reconverged; request normal keyframe recovery once. + candidate->pendingKey = true; + } + } + } + bool cached = group_->attempted == ts; + bool continuous = cached ? last_ == group_->previous : last_ == group_->timestamp; + if (!owner_.unsafe_skip_ && !continuous && !(cached && group_->image && + group_->image->_frameType == webrtc::VideoFrameType::kVideoFrameKey)) { + group_.reset(); + group_ = FindOrCreate(true, ts); + cached = group_->attempted == ts; + key = true; + ++owner_.dependencySplits; + } + if (key && cached && group_->image && + group_->image->_frameType != webrtc::VideoFrameType::kVideoFrameKey) { + group_.reset(); + group_ = NewGroup(); + cached = false; + } + if (before != group_) { + ++owner_.splits; + last_.reset(); + key = true; + } + if (cached) { + ++owner_.hits; + } else { + key = key || group_->pendingKey; + group_->pendingKey = false; + group_->previous = group_->timestamp; + group_->attempted = ts; + group_->image.reset(); + group_->dropped = false; + const std::vector types{ + key ? webrtc::VideoFrameType::kVideoFrameKey : webrtc::VideoFrameType::kVideoFrameDelta}; + ++owner_.encodes; + group_->UpdateCodecRates(); + const int64_t latency_us = static_cast(owner_.simulatedLatencyMs.load()) * + 1000 * group_->config.width * group_->config.height / (640 * 360); + if (latency_us > 0) std::this_thread::sleep_for(std::chrono::microseconds(latency_us)); + result = group_->codec->Encode(frame, &types); + if (group_->image) group_->timestamp = ts; + } + output = group_->image; + info = group_->info; + dropped = group_->dropped; + if (output) last_ = ts; + } + if (output) { + output->SetRtpTimestamp(frame.rtp_timestamp()); + ++owner_.callbacks; + callback_->OnEncodedImage(*output, &info); + } else if (dropped) { + callback_->OnFrameDropped(frame.rtp_timestamp(), 0, true); + } + return result; + } + + private: + bool Compatible(const Group& group) const { + // Upstream DefaultTemporalLayers ignores RTT/loss for this L1T1 codec. + return SameConfig(group, *config_, *settings_) && group.rates == rates_; + } + void ApplyRates(Group& group) { + group.rates = rates_; + group.UpdateCodecRates(); + group.codec->OnPacketLossRateUpdate(loss_); + group.codec->OnRttUpdate(rtt_); + group.loss = loss_; + group.rtt = rtt_; + } + std::shared_ptr NewGroup() { + std::erase_if(owner_.groups_, [](const auto& group) { return group.expired(); }); + auto group = std::make_shared(owner_, env_, *config_, *settings_); + ApplyRates(*group); + owner_.groups_.push_back(group); + return group; + } + std::shared_ptr FindOrCreate(bool encoding, uint32_t ts) { + if (owner_.pooled_) { + for (auto it = owner_.groups_.begin(); it != owner_.groups_.end();) { + auto group = it->lock(); + if (!group) { it = owner_.groups_.erase(it); continue; } + ++it; + if (!Compatible(*group)) continue; + if (!encoding && !group->timestamp) return group; + if (encoding && group->attempted == ts && group->image && + group->image->_frameType == webrtc::VideoFrameType::kVideoFrameKey) return group; + if (encoding && !group->timestamp) return group; + } + } + return NewGroup(); + } + PoolFactory& owner_; + webrtc::Environment env_; + std::unique_ptr metadata_; + std::optional config_; + std::optional settings_; + std::optional rates_; + float loss_ = 0; + int64_t rtt_ = 0; + std::shared_ptr group_; + std::optional last_; + webrtc::EncodedImageCallback* callback_ = nullptr; + }; + + public: + std::unique_ptr Create( + const webrtc::Environment& env, const webrtc::SdpVideoFormat&) override { + return std::make_unique(*this, env); + } + + private: + bool pooled_; + bool group_adjuster_; + bool unsafe_skip_; + std::unique_ptr builtin_; + webrtc::SdpVideoFormat format_; + std::mutex mutex_; + std::vector> groups_; +}; diff --git a/package-lock.json b/package-lock.json index 4b9d835..2f2e242 100644 --- a/package-lock.json +++ b/package-lock.json @@ -15,8 +15,6 @@ "@types/sdp-transform": "3.0.0", "@types/ws": "8.18.1", "@vitejs/plugin-react": "6.0.5", - "livekit-client": "2.22.1", - "livekit-server-sdk": "2.18.0", "react": "19.2.8", "react-dom": "19.2.8", "sdp-transform": "3.0.0", @@ -31,13 +29,6 @@ "node": ">=24.0.0 <27" } }, - "node_modules/@bufbuild/protobuf": { - "version": "1.10.1", - "resolved": "https://registry.npmjs.org/@bufbuild/protobuf/-/protobuf-1.10.1.tgz", - "integrity": "sha512-wJ8ReQbHxsAfXhrf9ixl0aYbZorRuOWpBNzm8pL8ftmSxQx/wnJD5Eg861NwJU/czy2VXFIebCeZnZrI9rktIQ==", - "dev": true, - "license": "(Apache-2.0 AND BSD-3-Clause)" - }, "node_modules/@esbuild/aix-ppc64": { "version": "0.28.2", "resolved": "https://registry.npmjs.org/@esbuild/aix-ppc64/-/aix-ppc64-0.28.2.tgz", @@ -487,23 +478,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@livekit/mutex": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/@livekit/mutex/-/mutex-1.1.1.tgz", - "integrity": "sha512-EsshAucklmpuUAfkABPxJNhzj9v2sG7JuzFDL4ML1oJQSV14sqrpTYnsaOudMAw9yOaW53NU3QQTlUQoRs4czw==", - "dev": true, - "license": "Apache-2.0" - }, - "node_modules/@livekit/protocol": { - "version": "1.50.4", - "resolved": "https://registry.npmjs.org/@livekit/protocol/-/protocol-1.50.4.tgz", - "integrity": "sha512-L1uggNQAqyY21smQY8AllyOYbcv9Me9TaxwuLytL1R8ck9nbYPmQLNwEDi3pOFGAMa5F8I2nUi2Jc59W5awxlA==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@bufbuild/protobuf": "^1.10.0" - } - }, "node_modules/@oxc-project/types": { "version": "0.144.0", "resolved": "https://registry.npmjs.org/@oxc-project/types/-/types-0.144.0.tgz", @@ -802,14 +776,6 @@ "dev": true, "license": "MIT" }, - "node_modules/@types/dom-mediacapture-record": { - "version": "1.0.22", - "resolved": "https://registry.npmjs.org/@types/dom-mediacapture-record/-/dom-mediacapture-record-1.0.22.tgz", - "integrity": "sha512-mUMZLK3NvwRLcAAT9qmcK+9p7tpU2FHdDsntR3YI4+GY88XrgG4XiE7u1Q2LAN2/FZOz/tdMDC3GQCR4T8nFuw==", - "dev": true, - "license": "MIT", - "peer": true - }, "node_modules/@types/estree": { "version": "1.0.9", "resolved": "https://registry.npmjs.org/@types/estree/-/estree-1.0.9.tgz", @@ -1106,16 +1072,6 @@ "@types/estree": "^1.0.0" } }, - "node_modules/events": { - "version": "3.3.0", - "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", - "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=0.8.x" - } - }, "node_modules/expect-type": { "version": "1.4.0", "resolved": "https://registry.npmjs.org/expect-type/-/expect-type-1.4.0.tgz", @@ -1159,16 +1115,6 @@ "node": "^8.16.0 || ^10.6.0 || >=11.0.0" } }, - "node_modules/jose": { - "version": "6.2.9", - "resolved": "https://registry.npmjs.org/jose/-/jose-6.2.9.tgz", - "integrity": "sha512-XrchZOFZUl/T3vTwRe8XK+cJrGtMF4th1ARnDfwbBXFKThGhlsxEE4Zu03AD/bjJSt/9jT/mxrOCkJWOg77aPA==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/panva" - } - }, "node_modules/lightningcss": { "version": "1.33.0", "resolved": "https://registry.npmjs.org/lightningcss/-/lightningcss-1.33.0.tgz", @@ -1442,97 +1388,6 @@ "url": "https://opencollective.com/parcel" } }, - "node_modules/livekit-client": { - "version": "2.22.1", - "resolved": "https://registry.npmjs.org/livekit-client/-/livekit-client-2.22.1.tgz", - "integrity": "sha512-kCxTlGOV2XWYXZ7zK01UUKjgpVydvJ0CBgdHuv2I+s4q08tn0UTzpHfFqXPb5Aba4h6KdNU0TjQCHzv4Lww7fQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@livekit/mutex": "1.1.1", - "@livekit/protocol": "1.50.4", - "events": "^3.3.0", - "jose": "^6.1.0", - "loglevel": "^1.9.2", - "machina": "^7.0.0", - "sdp-transform": "^2.15.0", - "tslib": "2.8.1", - "typed-emitter": "^2.1.0", - "webrtc-adapter": "9.0.6" - }, - "peerDependencies": { - "@types/dom-mediacapture-record": "^1" - } - }, - "node_modules/livekit-client/node_modules/sdp-transform": { - "version": "2.15.0", - "resolved": "https://registry.npmjs.org/sdp-transform/-/sdp-transform-2.15.0.tgz", - "integrity": "sha512-KrOH82c/W+GYQ0LHqtr3caRpM3ITglq3ljGUIb8LTki7ByacJZ9z+piSGiwZDsRyhQbYBOBJgr2k6X4BZXi3Kw==", - "dev": true, - "license": "MIT", - "bin": { - "sdp-verify": "checker.js" - } - }, - "node_modules/livekit-server-sdk": { - "version": "2.18.0", - "resolved": "https://registry.npmjs.org/livekit-server-sdk/-/livekit-server-sdk-2.18.0.tgz", - "integrity": "sha512-iUGmkbyi0g4DHKQ0Z4VSVs/V+ddp3T75ccQIe6LX94AYlqgyh+SVSM1lQMjFQuI3Qh6Z9v06T4xxV+vWmp7VeQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@bufbuild/protobuf": "^1.10.1", - "@livekit/protocol": "1.48.0", - "jose": "^5.1.2" - }, - "engines": { - "node": ">=18" - } - }, - "node_modules/livekit-server-sdk/node_modules/@livekit/protocol": { - "version": "1.48.0", - "resolved": "https://registry.npmjs.org/@livekit/protocol/-/protocol-1.48.0.tgz", - "integrity": "sha512-fYHYgltH6YavAsokl3qsHLkBdQeKCl4UORVTub5crS3t8JtKFZ0uinHDFQ+XXdNKS6Ub9gcOjV+UHcDiqnWXoQ==", - "dev": true, - "license": "Apache-2.0", - "dependencies": { - "@bufbuild/protobuf": "^1.10.0" - } - }, - "node_modules/livekit-server-sdk/node_modules/jose": { - "version": "5.10.0", - "resolved": "https://registry.npmjs.org/jose/-/jose-5.10.0.tgz", - "integrity": "sha512-s+3Al/p9g32Iq+oqXxkW//7jk2Vig6FF1CFqzVXoTUXt2qz89YWbL+OwS17NFYEvxC35n0FKeGO2LGYSxeM2Gg==", - "dev": true, - "license": "MIT", - "funding": { - "url": "https://github.com/sponsors/panva" - } - }, - "node_modules/loglevel": { - "version": "1.9.2", - "resolved": "https://registry.npmjs.org/loglevel/-/loglevel-1.9.2.tgz", - "integrity": "sha512-HgMmCqIJSAKqo68l0rS2AanEWfkxaZ5wNiEFb5ggm08lDs9Xl2KxBlX3PTcaD2chBM1gXAYf491/M2Rv8Jwayg==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">= 0.6.0" - }, - "funding": { - "type": "tidelift", - "url": "https://tidelift.com/funding/github/npm/loglevel" - } - }, - "node_modules/machina": { - "version": "7.0.1", - "resolved": "https://registry.npmjs.org/machina/-/machina-7.0.1.tgz", - "integrity": "sha512-0Xkgs8NeBvO2hW6bjJT8ewLWrMfh1haMPstaghpASWjXkLjRhtDiaJG1Kb7mslaOHxZYxpH7PIjpmirRVYbMMA==", - "dev": true, - "license": "MIT", - "engines": { - "node": ">=22.22" - } - }, "node_modules/magic-string": { "version": "0.30.21", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.21.tgz", @@ -1688,17 +1543,6 @@ "@rolldown/binding-win32-x64-msvc": "1.2.4" } }, - "node_modules/rxjs": { - "version": "7.8.2", - "resolved": "https://registry.npmjs.org/rxjs/-/rxjs-7.8.2.tgz", - "integrity": "sha512-dhKf903U/PQZY6boNNtAGdWbG85WAbjT/1xYoZIC7FAY0yWapOBQVsVrDl58W86//e1VpMNBtRV4MaXfdMySFA==", - "dev": true, - "license": "Apache-2.0", - "optional": true, - "dependencies": { - "tslib": "^2.1.0" - } - }, "node_modules/scheduler": { "version": "0.27.0", "resolved": "https://registry.npmjs.org/scheduler/-/scheduler-0.27.0.tgz", @@ -1706,13 +1550,6 @@ "dev": true, "license": "MIT" }, - "node_modules/sdp": { - "version": "3.2.2", - "resolved": "https://registry.npmjs.org/sdp/-/sdp-3.2.2.tgz", - "integrity": "sha512-xZocWwfyp4hkbN4hLWxMjmv2Q8aNa9MhmOZ7L9aCZPT+dZsgRr6wZRrSYE3HTdyk/2pZKPSgqI7ns7Een1xMSA==", - "dev": true, - "license": "MIT" - }, "node_modules/sdp-transform": { "version": "3.0.0", "resolved": "https://registry.npmjs.org/sdp-transform/-/sdp-transform-3.0.0.tgz", @@ -1798,13 +1635,6 @@ "node": ">=14.0.0" } }, - "node_modules/tslib": { - "version": "2.8.1", - "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.8.1.tgz", - "integrity": "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w==", - "dev": true, - "license": "0BSD" - }, "node_modules/tsx": { "version": "4.23.12", "resolved": "https://registry.npmjs.org/tsx/-/tsx-4.23.12.tgz", @@ -1824,16 +1654,6 @@ "fsevents": "~2.3.3" } }, - "node_modules/typed-emitter": { - "version": "2.1.0", - "resolved": "https://registry.npmjs.org/typed-emitter/-/typed-emitter-2.1.0.tgz", - "integrity": "sha512-g/KzbYKbH5C2vPkaXGu8DJlHrGKHLsM25Zg9WuC9pMGfuvT+X25tZQWo5fK1BjBm8+UrVE9LDCvaY0CQk+fXDA==", - "dev": true, - "license": "MIT", - "optionalDependencies": { - "rxjs": "*" - } - }, "node_modules/typescript": { "version": "6.0.3", "resolved": "https://registry.npmjs.org/typescript/-/typescript-6.0.3.tgz", @@ -2023,20 +1843,6 @@ } } }, - "node_modules/webrtc-adapter": { - "version": "9.0.6", - "resolved": "https://registry.npmjs.org/webrtc-adapter/-/webrtc-adapter-9.0.6.tgz", - "integrity": "sha512-CHbl2ZQbxx164IgWRgzJno4hWtM4tFbRam1QfI3Yxhs3w/DvqluVxVWeXs3oL5/fbGkSNLKo0Ty5MgUWceNhog==", - "dev": true, - "license": "BSD-3-Clause", - "dependencies": { - "sdp": "^3.2.0" - }, - "engines": { - "node": ">=6.0.0", - "npm": ">=3.10.0" - } - }, "node_modules/why-is-node-running": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/why-is-node-running/-/why-is-node-running-2.3.0.tgz", diff --git a/package.json b/package.json index 43a343c..9b6c33a 100644 --- a/package.json +++ b/package.json @@ -20,7 +20,7 @@ "gate:client-media": "tsx scripts/client-media-gate.ts", "gate:client-native-host": "tsx scripts/client-native-host-gate.ts", "gate:client-browser-fanout": "tsx scripts/client-browser-fanout-gate.ts", - "gate:client-native-sfu": "tsx scripts/client-native-sfu-gate.ts", + "gate:embedded-sfu": "tsx scripts/embedded-sfu-gate.ts", "gate:client-link": "tsx scripts/client-link-gate.ts", "gate:peer-topology-loopback": "tsx scripts/peer-assisted-benchmark.ts", "gate:sfu-admission": "vitest run tests/sfu-client.test.ts tests/route-transition.test.ts && go test ./internal/server/...", @@ -40,8 +40,6 @@ "@types/sdp-transform": "3.0.0", "@types/ws": "8.18.1", "@vitejs/plugin-react": "6.0.5", - "livekit-client": "2.22.1", - "livekit-server-sdk": "2.18.0", "react": "19.2.8", "react-dom": "19.2.8", "sdp-transform": "3.0.0", diff --git a/scripts/assemble-client.mjs b/scripts/assemble-client.mjs index 55cea75..48e45ce 100644 --- a/scripts/assemble-client.mjs +++ b/scripts/assemble-client.mjs @@ -10,7 +10,6 @@ import { existsSync, lstatSync, mkdirSync, - mkdtempSync, openSync, readSync, readFileSync, @@ -18,14 +17,14 @@ import { rmSync, writeFileSync, } from "node:fs"; -import { tmpdir } from "node:os"; import { dirname, isAbsolute, join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { clientPackageTarget, CLOUDFLARED_VERSION } from "./client-package-targets.mjs"; +import { clientPackageTarget, clientGoEnvironment, CLOUDFLARED_VERSION } from "./client-package-targets.mjs"; import { writeClientPlatformAssets } from "./client-icons.mjs"; import { writeClientLicenseNotices } from "./package-licenses.mjs"; import { tarExecutable } from "./archive-tool.mjs"; +import { resetBuildWorkspace } from "./build-workspace.mjs"; function fail(message) { throw new Error(message); @@ -130,10 +129,20 @@ function readDescriptor(path) { }; } -function assertOutsideRepository(repositoryRoot, outputRoot) { +function assertOutputDirectory(repositoryRoot, outputRoot, target) { const path = relative(repositoryRoot, outputRoot); - if (path === "" || (path.split(/[\\/]/)[0] !== ".." && !isAbsolute(path))) { - fail("Client output directory must be outside the repository"); + const candidateOutput = join(repositoryRoot, "build", "client-package", target.id, + "candidate", `Screener-Client-${target.id}`); + if (outputRoot !== candidateOutput && + (path === "" || (path.split(/[\\/]/)[0] !== ".." && !isAbsolute(path)))) { + fail("Client output must be outside the repository or its exact candidate workspace"); + } + if (outputRoot === candidateOutput) { + for (let parent = dirname(outputRoot); parent !== repositoryRoot; parent = dirname(parent)) { + const metadata = lstatSync(parent, { throwIfNoEntry: false }); + if (metadata && (!metadata.isDirectory() || metadata.isSymbolicLink() || + realpathSync(parent) !== parent)) fail("Client candidate output must not contain links"); + } } if (existsSync(outputRoot)) { fail("Client output directory must not already exist"); @@ -172,13 +181,16 @@ for (let index = 0; index < options.length; index += 2) { } const target = clientPackageTarget(targetArgument); if (!target) fail("Client package target is invalid"); +if (target.cgo && process.platform !== target.nodePlatform) { + fail(`Client target ${target.id} requires a native macOS runner with its SDK and cgo`); +} const repositoryRoot = realpathSync(resolve(dirname(fileURLToPath(import.meta.url)), "..")); const descriptorPath = realpathSync(resolve(positional[0])); const outputRoot = resolve(process.cwd(), positional[1]); const capturePath = captureArgument ? realpathSync(resolve(captureArgument)) : null; const tunnelPath = tunnelArgument ? realpathSync(resolve(tunnelArgument)) : null; -assertOutsideRepository(repositoryRoot, outputRoot); +assertOutputDirectory(repositoryRoot, outputRoot, target); if (capturePath && !target.captureName) { fail("Capture runtime is invalid for the Client package target"); } @@ -198,7 +210,7 @@ if (!existsSync(artifactPath) || sha256(artifactPath) !== descriptor.artifactSha fail("Application release artifact does not match its descriptor"); } -const temporaryRoot = mkdtempSync(join(tmpdir(), `screener-client-${revision.slice(0, 7)}-`)); +const temporaryRoot = resetBuildWorkspace(repositoryRoot, "client-package", target.id, "assembly"); const packageRoot = join(temporaryRoot, "package"); const releaseRoot = join(temporaryRoot, "release"); try { @@ -245,12 +257,7 @@ try { "-o", clientPath, "./cmd/screener-client", - ], repositoryRoot, { - ...process.env, - GOOS: target.goos, - GOARCH: target.goarch, - CGO_ENABLED: "0", - }); + ], repositoryRoot, clientGoEnvironment(target)); chmodSync(clientPath, 0o755); assertTargetExecutable(clientPath, target, "Client executable"); writeClientLicenseNotices(repositoryRoot, packageRoot, goCommand, target, diff --git a/scripts/browser-encoded-fanout-probe.mjs b/scripts/browser-encoded-fanout-probe.mjs new file mode 100644 index 0000000..1fb1e6d --- /dev/null +++ b/scripts/browser-encoded-fanout-probe.mjs @@ -0,0 +1,143 @@ +import http from 'node:http'; +import { mkdirSync, writeFileSync } from 'node:fs'; + +// Explicit local experiment, never part of product startup or ordinary CI. +const codec = process.argv.includes('--h264') ? 'H264' : 'VP8'; +const lateJoin = process.argv.includes('--late-join'); +const limited = process.argv.includes('--limit-secondary'); +const limitedPrimary = process.argv.includes('--limit-primary'); +const requestKeyframe = process.argv.includes('--request-keyframe'); +const ordinary = process.argv.includes('--ordinary'); +const standard = process.argv.includes('--standard'); +const cloneOwn = process.argv.includes('--clone-own'); +const durationMs = limitedPrimary ? 14000 : 7000; +const suffix = (lateJoin ? '-late' : '') + (limited ? '-limited' : '') + + (limitedPrimary ? '-primary-limited' : '') + (requestKeyframe ? '-key' : '') + (ordinary ? '-ordinary' : '') + + (standard ? '-standard' : '') + (cloneOwn ? '-clone-own' : ''); +const worker = `const writers=new Map(); +self.onrtctransform=event=>{ + const {readable,writable,options}=event.transformer; + const writer=writable.getWriter();writers.set(options.name,writer); + readable.pipeTo(new WritableStream({async write(frame){ + if(options.name!=='A')return; + const copy=new RTCEncodedVideoFrame(frame); + await writer.write(${cloneOwn ? 'new RTCEncodedVideoFrame(frame)' : 'frame'}); + if(writers.has('B'))await writers.get('B').write(copy); + }})).catch(error=>self.postMessage({error:error.name+': '+error.message})); +};`; +mkdirSync('build/encoder-pool', { recursive: true }); +const html = `Encoded fanout probe + + +
starting
`; + +const timer = setTimeout(()=>server.close(),60000); +const server = http.createServer((req,res)=>{ + if(req.url==='/worker.js'){res.setHeader('content-type','text/javascript');res.end(worker);} + else if(req.url==='/result'&&req.method==='POST'){ + let data='';req.on('data',chunk=>{data+=chunk;if(data.length>65536)req.destroy();}); + req.on('end',()=>{const result=JSON.parse(data);writeFileSync('build/encoder-pool/browser-fanout-'+codec.toLowerCase()+suffix+'.json',JSON.stringify(result,null,2)+'\n');console.log(JSON.stringify(result));res.end('ok');clearTimeout(timer);server.close();}); + } else {res.setHeader('content-type','text/html');res.end(html);} +}); +server.listen(0,'127.0.0.1',()=>console.log('http://127.0.0.1:'+server.address().port)); diff --git a/scripts/build-workspace.mjs b/scripts/build-workspace.mjs new file mode 100644 index 0000000..452cb2c --- /dev/null +++ b/scripts/build-workspace.mjs @@ -0,0 +1,21 @@ +import { lstatSync, mkdirSync, realpathSync, rmSync } from "node:fs"; +import { join } from "node:path"; + +export function resetBuildWorkspace(repositoryRoot, ...parts) { + if (parts.length < 2 || parts.some((part) => typeof part !== "string" || !/^[a-z0-9][a-z0-9-]*$/.test(part))) { + throw new Error("Build workspace requires a named descendant of build"); + } + let directory = realpathSync(repositoryRoot); + for (const part of ["build", ...parts]) { + directory = join(directory, part); + const metadata = lstatSync(directory, { throwIfNoEntry: false }); + if (metadata && (!metadata.isDirectory() || metadata.isSymbolicLink() || + realpathSync(directory) !== directory)) { + throw new Error("Build workspace must not contain links"); + } + if (!metadata) mkdirSync(directory, { mode: 0o700 }); + } + rmSync(directory, { recursive: true, force: true }); + mkdirSync(directory, { mode: 0o700 }); + return directory; +} diff --git a/scripts/check-client.mjs b/scripts/check-client.mjs index 2e41d2a..45a1d5a 100644 --- a/scripts/check-client.mjs +++ b/scripts/check-client.mjs @@ -9,7 +9,7 @@ import { import { basename, dirname, join, resolve } from "node:path"; import { fileURLToPath } from "node:url"; -import { CLIENT_PACKAGE_TARGETS, clientPackageTarget } from "./client-package-targets.mjs"; +import { CLIENT_PACKAGE_TARGETS, clientPackageTarget, clientGoEnvironment } from "./client-package-targets.mjs"; // Every Go command embeds the Vite output, so the build, vet and test steps all // fail without it. The Hosted binary is cross-built for its deployment target. @@ -65,7 +65,14 @@ function checkCore() { const buildRoot = join(root, "build", "client-check"); mkdirSync(buildRoot, { recursive: true }); - const builds = CLIENT_PACKAGE_TARGETS.flatMap((target) => + const buildTargets = CLIENT_PACKAGE_TARGETS.filter((target) => { + if (target.cgo && process.platform !== target.nodePlatform) { + process.stderr.write(`Skipped ${target.id}: its cgo dependencies require a native macOS runner and SDK; Darwin acceptance remains pending.\n`); + return false; + } + return true; + }); + const builds = buildTargets.flatMap((target) => ["screener-client", "screener-peer-gate"].map((command) => ({ target, command })), ); builds.push({ target: SERVER_TARGET, command: "screener-server" }); @@ -74,12 +81,7 @@ function checkCore() { ? `${command}.exe` : `${command}-${target.id}`; run(go, ["build", "-trimpath", "-o", join(buildRoot, outputName), `./cmd/${command}`], { - env: { - ...process.env, - GOOS: target.goos, - GOARCH: target.goarch, - CGO_ENABLED: "0", - }, + env: clientGoEnvironment(target), }); } } @@ -90,32 +92,19 @@ function runClientTests(go) { return; } - // Windows associates its listen prompt with the test executable path. Keep - // the one UDP integration package at a stable path so repeated checks do not - // create a new firewall rule for every Go build directory. - const packages = run(go, ["list", "./..."], { capture: true }) + // Windows firewall permissions follow executable paths, including tests that + // open sockets indirectly. Never execute a test from Go's temporary directory. + const packages = run(go, ["list", "-f", '{{if or .TestGoFiles .XTestGoFiles}}[{{printf "%q" .ImportPath}},{{printf "%q" .Dir}}]{{end}}', "./..."], { capture: true }) .split(/\r?\n/) .map((value) => value.trim()) - .filter(Boolean); - const stableNetworkPackages = [ - { - package: "github.com/TNTcraftHIM/Screener/internal/client/mediaedge", - binary: "mediaedge.test.exe", - }, - ]; - const stablePackageNames = new Set( - stableNetworkPackages.map((entry) => entry.package), - ); - const otherPackages = packages.filter((value) => !stablePackageNames.has(value)); - if (otherPackages.length > 0) { - run(go, ["test", ...otherPackages]); - } + .filter(Boolean) + .map((value) => JSON.parse(value)); const stableRoot = join(root, "build", "client-check"); mkdirSync(stableRoot, { recursive: true }); - for (const entry of stableNetworkPackages) { - const binary = join(stableRoot, entry.binary); - run(go, ["test", "-c", "-o", binary, entry.package]); - run(binary, ["-test.v"]); + for (const [entry, directory] of packages) { + const binary = join(stableRoot, `${basename(entry)}.test.exe`); + run(go, ["test", "-c", "-o", binary, entry]); + run(binary, [], { cwd: directory }); } } @@ -175,7 +164,7 @@ function checkPlatformCapture() { const probe = JSON.parse(raw); const expectedPlatform = process.platform === "win32" ? "windows" : process.platform; if ( - probe?.protocol !== 4 || + probe?.protocol !== 7 || probe.platform !== expectedPlatform || typeof probe.platformBuild !== "string" || typeof probe.videoCapture !== "boolean" || diff --git a/scripts/client-browser-fanout-gate.ts b/scripts/client-browser-fanout-gate.ts index 7a21d7c..d2be3c9 100644 --- a/scripts/client-browser-fanout-gate.ts +++ b/scripts/client-browser-fanout-gate.ts @@ -203,6 +203,7 @@ async function main(): Promise { stage = "share-picker"; await click(host, 'button.lr-tv-big.is-action'); stage = "share-browser-capture"; + await click(host, 'button[data-source-tab="browser"]'); await click(host, '.lr-source-option.is-browser'); stage = "share-ingress"; await waitForSample(() => read(host), (s) => s.sharing && s.receiveRequests === 1 && s.outbound === 1, 30000); diff --git a/scripts/client-media-gate.ts b/scripts/client-media-gate.ts index 425252c..6f51c18 100644 --- a/scripts/client-media-gate.ts +++ b/scripts/client-media-gate.ts @@ -151,6 +151,20 @@ function startPageServer(port: number) { }); } +function writeCaptureControl(child: ChildProcessWithoutNullStreams, command: string): void { + const payload = Buffer.from(command, "ascii"); + if (payload.length === 0 || payload.length > 64 || !/^[\x20-\x7e]+$/.test(command)) { + throw new Error("Capture control exceeds its SMED bound"); + } + const frame = Buffer.alloc(32 + payload.length); + frame.write("SMED", 0, "ascii"); + frame[4] = 2; + frame[5] = 7; + frame.writeUInt32BE(payload.length, 28); + payload.copy(frame, 32); + child.stdin.write(frame); +} + function observeCapture( child: ChildProcessWithoutNullStreams, evidence: CaptureEvidence, @@ -159,22 +173,22 @@ function observeCapture( let framesAtRequest = 0; child.stdout.on("data", (chunk: Buffer) => { buffered = Buffer.concat([buffered, chunk]); - while (buffered.length >= 28) { - if (buffered.subarray(0, 4).toString("ascii") !== "SMED" || buffered[4] !== 1) { + while (buffered.length >= 32) { + if (buffered.subarray(0, 4).toString("ascii") !== "SMED" || buffered[4] !== 2) { child.kill(); return; } const kind = buffered[5]!; const flags = buffered[6]!; - const size = buffered.readUInt32BE(24); - const maximum = kind === 3 ? 4 * 1024 : 1024 * 1024; - if (size === 0 || size > maximum) { + const size = buffered.readUInt32BE(28); + const maximum = kind === 3 ? 4 * 1024 : 4 * 1024 * 1024; + if ((size === 0 && kind !== 5) || size > maximum) { child.kill(); return; } - if (buffered.length < 28 + size) return; - const payload = buffered.subarray(28, 28 + size); - buffered = buffered.subarray(28 + size); + if (buffered.length < 32 + size) return; + const payload = buffered.subarray(32, 32 + size); + buffered = buffered.subarray(32 + size); if (kind === 3) { const state = JSON.parse(payload.toString("utf8")) as { state?: string }; evidence.starting ||= state.state === "starting"; @@ -184,7 +198,7 @@ function observeCapture( if (!evidence.keyFrameRequested && evidence.frames >= 5) { evidence.keyFrameRequested = true; framesAtRequest = evidence.frames; - child.stdin.write("K"); + writeCaptureControl(child, "K -1"); } else if (evidence.keyFrameRequested && evidence.frames > framesAtRequest && flags === 1) { evidence.recoveryFrame = true; } @@ -200,20 +214,20 @@ function observeAudioCapture( let buffered = Buffer.alloc(0); child.stdout.on("data", (chunk: Buffer) => { buffered = Buffer.concat([buffered, chunk]); - while (buffered.length >= 28) { - if (buffered.subarray(0, 4).toString("ascii") !== "SMED" || buffered[4] !== 1) { + while (buffered.length >= 32) { + if (buffered.subarray(0, 4).toString("ascii") !== "SMED" || buffered[4] !== 2) { child.kill(); return; } const kind = buffered[5]!; - const size = buffered.readUInt32BE(24); - const maximum = kind === 3 ? 4 * 1024 : 1024 * 1024; - if (size === 0 || size > maximum || buffered.length < 28 + size) { + const size = buffered.readUInt32BE(28); + const maximum = kind === 3 ? 4 * 1024 : 4 * 1024 * 1024; + if (size === 0 || size > maximum || buffered.length < 32 + size) { if (size === 0 || size > maximum) child.kill(); return; } - const payload = buffered.subarray(28, 28 + size); - buffered = buffered.subarray(28 + size); + const payload = buffered.subarray(32, 32 + size); + buffered = buffered.subarray(32 + size); if (kind === 3) { const state = JSON.parse(payload.toString("utf8")) as { state?: string; @@ -237,7 +251,7 @@ async function stopCapture(child: ChildProcessWithoutNullStreams): Promise if (child.exitCode !== 0) throw new Error("Native capture exited unsuccessfully"); return; } - child.stdin.write("\n"); + writeCaptureControl(child, "Q"); const code = await withDeadline( () => new Promise((resolveExit) => child.once("exit", resolveExit)), Date.now() + 3_000, @@ -341,7 +355,7 @@ async function browserMediaGate(input: { rejectRequest(new Error("Native request timed out: " + type)); }, 8_000); pending.set(id, { resolve: resolveRequest, reject: rejectRequest, timer }); - socket!.send(JSON.stringify({ version: 8, id, type, ...fields })); + socket!.send(JSON.stringify({ version: 9, id, type, ...fields })); }); }; socket.onmessage = (event) => { @@ -724,7 +738,7 @@ async function main(): Promise { const probe = JSON.parse(run(executable, ["--probe"])) as Probe; const adapter = probe.adapters.find((candidate) => candidate.hardwareH264.length > 0); const encoder = adapter?.hardwareH264[0]; - if (probe.protocol !== 4 || !adapter || !encoder) { + if (probe.protocol !== 7 || !adapter || !encoder) { throw new Error("No hardware H264 capture path is available"); } @@ -802,7 +816,8 @@ async function main(): Promise { "balanced", "--codec", "h264", - "--protocol-v4", + "--protocol-v7", + "--output", "1280", "720", "30", "3000000", ], { stdio: "pipe", windowsHide: true }); capture.stderr.resume(); observeCapture(capture, evidence); diff --git a/scripts/client-native-host-gate.ts b/scripts/client-native-host-gate.ts index 01f2509..db5892d 100644 --- a/scripts/client-native-host-gate.ts +++ b/scripts/client-native-host-gate.ts @@ -6,7 +6,7 @@ import { } from "node:child_process"; import { randomBytes } from "node:crypto"; import { createServer } from "node:http"; -import { mkdir, mkdtemp } from "node:fs/promises"; +import { mkdir, mkdtemp, readFile, stat } from "node:fs/promises"; import { tmpdir } from "node:os"; import { join, resolve } from "node:path"; import { pathToFileURL } from "node:url"; @@ -36,6 +36,15 @@ type VideoCodec = "h264" | "vp8"; const CODEC_PROBE = `(() => { const peers = [], BrowserPeer = RTCPeerConnection; + const sockets = [], BrowserSocket = WebSocket; + window.WebSocket = class extends BrowserSocket { + constructor(...args) { super(...args); sockets.push(this); } + }; + window.__screenerGatePublicSignal = () => sockets.some((socket) => { + const url = new URL(socket.url); + return socket.readyState === WebSocket.OPEN && url.protocol === 'wss:' && + url.host === location.host && url.pathname === '/signal'; + }); window.RTCPeerConnection = class extends BrowserPeer { constructor(...args) { super(...args); peers.push(this); } }; @@ -68,6 +77,7 @@ interface GateResult { livePresetChanges: number; liveQualityChanged: boolean; pausedQualityChanged: boolean; + backgroundProfileRecovery: boolean | null; mediaObjectPreserved: boolean; nativeSourceChanged: boolean; nativeQualityEvidence: boolean; @@ -87,6 +97,9 @@ interface GateResult { replacementViewerFrames: number | null; clientCrashEndedShare: boolean | null; mode: GateMode; + viewerLocation: "local-browser" | "public-url-browser" | "remote-peer"; + publicViewerPage: boolean; + publicViewerSignal: boolean; cleanup: Awaited>; error: string | null; stage?: string; @@ -563,7 +576,9 @@ async function main(): Promise { if (requestedCodec !== "auto" && requestedCodec !== "h264" && requestedCodec !== "vp8") { throw new Error("SCREENER_CLIENT_NATIVE_HOST_CODEC must be auto, h264, or vp8"); } - const remote = mode === "local" ? null : remoteOptions(); + const remote = mode === "cross-nat" || mode === "one-link" && + Boolean(process.env.SCREENER_REMOTE_HOST?.trim() || process.env.SCREENER_REMOTE_SSH_KEY?.trim()) + ? remoteOptions() : null; const chromePath = process.env.CHROME_PATH?.trim(); if (!chromePath) throw new Error("CHROME_PATH is required"); const go = process.env.SCREENER_GO?.trim() || "go"; @@ -574,6 +589,7 @@ async function main(): Promise { const profile = await mkdtemp(join(tmpdir(), "screener-client-media-")); const sourceProfile = await mkdtemp(join(tmpdir(), "screener-client-media-")); await mkdir(BUILD_ROOT, { recursive: true }); + const diagnosticDirectory = await mkdtemp(join(BUILD_ROOT, "native-host-gate-")); const sourcePort = await reservePort(); const appPort = await reservePort(); const debugPort = await reservePort(); @@ -590,7 +606,6 @@ async function main(): Promise { let chrome: ChildProcessWithoutNullStreams | null = null; let cdp: CdpConnection | null = null; let clientPort = 0; - let clientDiagnostics = ""; let remoteTunnel: ChildProcess | null = null; let stage = "setup"; const result: GateResult = { @@ -605,6 +620,7 @@ async function main(): Promise { livePresetChanges: 0, liveQualityChanged: false, pausedQualityChanged: false, + backgroundProfileRecovery: mode === "local" && sourceKind === "window" ? false : null, mediaObjectPreserved: false, nativeSourceChanged: false, nativeQualityEvidence: false, @@ -624,6 +640,9 @@ async function main(): Promise { replacementViewerFrames: mode === "local" ? 0 : null, clientCrashEndedShare: crashGate ? false : null, mode, + viewerLocation: remote ? "remote-peer" : mode === "one-link" ? "public-url-browser" : "local-browser", + publicViewerPage: false, + publicViewerSignal: false, cleanup: { browserExited: false, nativeExited: false, @@ -641,15 +660,17 @@ async function main(): Promise { stage = "source-server"; source = await sourceServer(sourcePort); stage = "capture-build"; + process.stderr.write(`${JSON.stringify({ stage, status: "started", at: new Date().toISOString() })}\n`); run(powershell(), [ "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", join(ROOT, "native", "capture", "windows", "build.ps1"), "-OutputDirectory", captureBuild, ]); + process.stderr.write(`${JSON.stringify({ stage, status: "finished", at: new Date().toISOString() })}\n`); stage = "client-build"; run(go, [ - "build", "-trimpath", "-o", clientBinary, "./cmd/screener-client", - ], ROOT); + "build", "-p", "1", "-trimpath", "-o", clientBinary, "./cmd/screener-client", + ], ROOT, { ...process.env, GOMAXPROCS: "2" }); if (remote) { stage = "remote-peer-build"; run( @@ -676,12 +697,13 @@ async function main(): Promise { "--capture-process", captureBinary, "--config", clientConfig, "--port", String(appPort), + "--debug", "--log-dir", diagnosticDirectory, ], { stdio: "pipe", windowsHide: true, env: { ...process.env, - SCREENER_DEBUG: "route", + SCREENER_DEBUG: "", SCREENER_CLIENT_GATE_NO_BROWSER: "true", // STUN_URLS reaches only the Client's own Pion edge: the in-process // room server never reads it, so the cross-NAT arm stays isolated. @@ -690,9 +712,7 @@ async function main(): Promise { : {}), }, }); - client.stderr.on("data", (chunk: Buffer) => { - clientDiagnostics = (clientDiagnostics + chunk.toString()).slice(-262_144); - }); + client.stderr.resume(); stage = "client-ready"; const clientInfo = await readClientEndpoint(client, mode === "one-link"); clientPort = clientInfo.endpoint.port; @@ -878,7 +898,17 @@ async function main(): Promise { } } else { const viewerURL = new URL(hostState.invite); - viewerURL.hostname = "localhost"; + if (mode === "local") viewerURL.hostname = "localhost"; + else { + if (viewerURL.protocol !== "https:" || viewerURL.origin !== clientInfo.publicOrigin) { + throw new Error("One-link invitation does not use the actual public origin"); + } + stage = "public-page-ready"; + await waitForValue(async (deadline) => { + const response = await fetch(viewerURL, { signal: AbortSignal.timeout(Math.max(1, deadline - Date.now())) }); + return response.ok; + }, Boolean, 20_000); + } stage = "viewer-page"; const viewer = await createPage(cdp, viewerURL.toString(), CODEC_PROBE, true); stage = "viewer-media"; @@ -906,6 +936,14 @@ async function main(): Promise { result.viewerWidth = viewerState.width; result.viewerHeight = viewerState.height; await assertVideoCodec(cdp, viewer, actualCodec); + if (mode === "one-link") { + result.codecPreserved = true; + result.publicViewerPage = await evaluate(cdp, viewer, + `location.origin === ${JSON.stringify(clientInfo.publicOrigin)} && location.pathname === ${JSON.stringify(viewerURL.pathname)}`, + Date.now() + 5_000); + result.publicViewerSignal = await evaluate(cdp, viewer, + "window.__screenerGatePublicSignal()", Date.now() + 5_000); + } else { await evaluate( cdp, viewer, @@ -1127,6 +1165,66 @@ async function main(): Promise { result.viewerWidth = resumed.width; result.viewerHeight = resumed.height; await assertVideoCodec(cdp, viewer, actualCodec); + if (sourceKind === "window") { + stage = "native-background-profile"; + const { targetInfos } = await sourceCdp!.call<{ + targetInfos: Array<{ targetId: string; type: string; url: string }>; + }>("Target.getTargets", {}, undefined, Date.now() + 5_000); + const target = targetInfos.find((value) => + value.type === "page" && value.url === `http://127.0.0.1:${sourcePort}/`, + ); + if (!target) throw new Error("Capture source page is missing"); + const { windowId } = await sourceCdp!.call<{ windowId: number }>( + "Browser.getWindowForTarget", { targetId: target.targetId }, + undefined, Date.now() + 5_000, + ); + await sourceCdp!.call("Browser.setWindowBounds", { + windowId, bounds: { windowState: "minimized" }, + }, undefined, Date.now() + 5_000); + try { + await evaluate(cdp, host, + `new Promise((resolve) => { + document.querySelector('button.lr-chip[aria-label="720p"]')?.click(); + requestAnimationFrame(() => requestAnimationFrame(resolve)); + })`, + Date.now() + 5_000, + ); + await waitForValue((deadline) => evaluate(cdp!, host, + `Boolean(document.querySelector('#host-advanced-door .lr-door-body[aria-busy="true"]'))`, + deadline, + ), Boolean, 5_000); + } finally { + await sourceCdp!.call("Browser.setWindowBounds", { + windowId, bounds: { windowState: "normal" }, + }, undefined, Date.now() + 5_000); + } + stage = "native-background-profile-recovery"; + // A quiet source retains its pending replacement until restoration. + // Prove its actual delivery before applying the following profile. + for (const [label, width, height] of [["720p", 1280, 720], ["480p", 854, 480]] as const) { + const before = await evaluate(cdp, viewer, + `document.querySelector('video')?.getVideoPlaybackQuality().totalVideoFrames ?? 0`, + Date.now() + 5_000, + ); + await evaluate(cdp, host, + `document.querySelector('button.lr-chip[aria-label="${label}"]')?.click()`, + Date.now() + 5_000, + ); + await waitForValue((deadline) => evaluate(cdp!, viewer, + `(() => { + const video = document.querySelector('video'); + return Boolean(video && video.srcObject === window.__screenerGateMedia && + video.videoWidth === ${width} && video.videoHeight === ${height} && + video.getVideoPlaybackQuality().totalVideoFrames >= ${before + 10}); + })()`, deadline, + ), Boolean, 20_000); + await waitForValue((deadline) => evaluate(cdp!, host, + `Boolean(document.querySelector('#host-advanced-door .lr-door-body[aria-busy="false"]'))`, + deadline, + ), Boolean, 5_000); + } + result.backgroundProfileRecovery = true; + } stage = "native-source-picker"; await waitForValue( (deadline) => evaluate( @@ -1191,8 +1289,22 @@ async function main(): Promise { await assertVideoCodec(cdp, viewer, actualCodec); result.codecPreserved = true; result.nativeQualityEvidence = await waitForValue( - async () => /"event":"sender-quality-evidence"[^\n]*"state":"(healthy|degraded)"/ - .test(clientDiagnostics), + async () => { + const logPath = join(diagnosticDirectory, "client.log"); + if ((await stat(logPath)).size > 8 * 1024 * 1024) { + throw new Error("Native Host gate diagnostic log exceeded its bound"); + } + const diagnostics = await readFile(logPath, "utf8"); + return diagnostics.split(/\r?\n/).some((line) => { + try { + const record = JSON.parse(line); + return record.msg === "screener-route" && record.event === "sender-quality-evidence" && + (record.state === "healthy" || record.state === "degraded"); + } catch { + return false; + } + }); + }, Boolean, 12_000, ); @@ -1300,6 +1412,7 @@ async function main(): Promise { result.replacementViewerConnected = null; result.replacementViewerFrames = null; } + } } } catch (error) { // Client stderr can contain implementation diagnostics or URLs; keep gate @@ -1343,12 +1456,16 @@ async function main(): Promise { result.passed = result.error === null && result.actualCodec !== null && result.preSharePreferenceEnabled && result.hostNativeActive && result.hostInvite && - (mode !== "local" + (remote ? result.remoteViewerConnected && result.remoteViewerPackets >= 30 && result.remoteNatPath && result.remotePeerExited === true && (mode !== "cross-nat" || result.reverseSignalTunnelClosed === true) - : result.viewerConnected && result.viewerFrames >= 30 && + : mode === "one-link" + ? result.publicViewerPage && result.publicViewerSignal && result.viewerConnected && + result.viewerFrames >= 30 && result.viewerWidth === 1920 && result.viewerHeight === 1080 && result.codecPreserved + : result.viewerConnected && result.viewerFrames >= 30 && result.qualityControlsEnabled && result.liveQualityChanged && + result.backgroundProfileRecovery !== false && result.livePresetChanges === 2 && result.codecPreserved && result.pausedQualityChanged && result.mediaObjectPreserved && diff --git a/scripts/client-native-sfu-gate.ts b/scripts/client-native-sfu-gate.ts deleted file mode 100644 index 6ad1d3e..0000000 --- a/scripts/client-native-sfu-gate.ts +++ /dev/null @@ -1,388 +0,0 @@ -import { - spawn, - spawnSync, - type ChildProcessWithoutNullStreams, -} from "node:child_process"; -import { randomBytes } from "node:crypto"; -import { createSocket } from "node:dgram"; -import { mkdtemp } from "node:fs/promises"; -import { tmpdir } from "node:os"; -import { join, resolve } from "node:path"; - -import { AccessToken } from "livekit-server-sdk"; -import { createServer as createViteServer, type ViteDevServer } from "vite"; - -import { - CdpConnection, - cleanupRun, - createPage, - evaluate, - launchChrome, - reservePort, - type PageHandle, - waitForVersion, -} from "./browser-gate-harness"; -import { - closeSourceBrowser, - powershell, - SOURCE_TITLE, - sourceServer, - startSourceBrowser, - waitForCaptureWindow, -} from "./client-native-host-gate"; -import { - readClientEndpoint, -} from "./client-gate-endpoint"; - -const ROOT = resolve(import.meta.dirname, ".."); -const BUILD_ROOT = join(ROOT, "build", "client-check"); - -function run(command: string, args: string[], cwd = ROOT): void { - const result = spawnSync(command, args, { - cwd, - stdio: ["ignore", "pipe", "pipe"], - encoding: "utf8", - }); - if (result.error) throw result.error; - if (result.status !== 0) { - throw new Error((result.stderr || result.stdout).trim() || `${command} failed`); - } -} - -async function reserveUDPPort(): Promise { - const socket = createSocket("udp4"); - return await new Promise((resolvePort, rejectPort) => { - socket.once("error", rejectPort); - socket.bind(0, "127.0.0.1", () => { - const address = socket.address(); - socket.close(() => resolvePort(address.port)); - }); - }); -} - -async function waitForHTTP(url: string, timeoutMs: number): Promise { - const deadline = Date.now() + timeoutMs; - while (Date.now() < deadline) { - try { - if ((await fetch(url)).ok) return; - } catch { - // The isolated service is still starting. - } - await new Promise((resolveWait) => setTimeout(resolveWait, 100)); - } - throw new Error(`service did not become ready: ${url}`); -} - -async function issueToken( - apiKey: string, - apiSecret: string, - room: string, - identity: string, - publish: boolean, -): Promise { - const token = new AccessToken(apiKey, apiSecret, { identity, ttl: 300 }); - token.addGrant({ - roomJoin: true, - room, - canPublish: publish, - canSubscribe: !publish, - canPublishData: false, - }); - return await token.toJwt(); -} - -async function stopProcess(process: ChildProcessWithoutNullStreams | null): Promise { - if (!process || process.exitCode !== null || process.signalCode !== null) return true; - process.kill(); - return await new Promise((resolveExit) => { - const timer = setTimeout(() => resolveExit(false), 5_000); - process.once("exit", () => { - clearTimeout(timer); - resolveExit(true); - }); - }); -} - -async function main(): Promise { - if (process.platform !== "win32") { - throw new Error("The native SFU gate currently requires Windows"); - } - if (process.env.SCREENER_CLIENT_NATIVE_SFU_GATE !== "true") { - throw new Error("SCREENER_CLIENT_NATIVE_SFU_GATE=true is required"); - } - const chromePath = process.env.CHROME_PATH?.trim(); - const livekitPath = process.env.SCREENER_LIVEKIT_SERVER?.trim(); - if (!chromePath || !livekitPath) { - throw new Error("CHROME_PATH and SCREENER_LIVEKIT_SERVER are required"); - } - const go = process.env.SCREENER_GO?.trim() || "go"; - const sourcePort = await reservePort(); - const sourceDebugPort = await reservePort(); - const vitePort = await reservePort(); - const debugPort = await reservePort(); - const livekitPort = await reservePort(); - const livekitTCPPort = await reservePort(); - const livekitUDPPort = await reserveUDPPort(); - const profile = await mkdtemp(join(tmpdir(), "screener-client-media-")); - const sourceProfile = await mkdtemp(join(tmpdir(), "screener-client-media-")); - const configPath = join(profile, "client.json"); - const captureBinary = join(BUILD_ROOT, "screener-client-capture.exe"); - const clientBinary = join(BUILD_ROOT, "screener-client.exe"); - const origin = `http://127.0.0.1:${vitePort}`; - const livekitURL = `ws://127.0.0.1:${livekitPort}`; - const apiKey = `gate${randomBytes(8).toString("hex")}`; - const apiSecret = randomBytes(32).toString("base64url"); - const room = `native-sfu-${randomBytes(8).toString("hex")}`; - const result = { - passed: false, - hostPublished: false, - viewerFrames: 0, - viewerWidth: 0, - viewerHeight: 0, - profileUpdated: false, - cleanup: false, - error: null as string | null, - stage: "setup", - }; - let vite: ViteDevServer | null = null; - let livekit: ChildProcessWithoutNullStreams | null = null; - let client: ChildProcessWithoutNullStreams | null = null; - let source: { close(): Promise } | null = null; - let sourceChrome: ChildProcessWithoutNullStreams | null = null; - let sourceCdp: CdpConnection | null = null; - let chrome: ChildProcessWithoutNullStreams | null = null; - let cdp: CdpConnection | null = null; - let hostPage: PageHandle | null = null; - let viewerPage: PageHandle | null = null; - let clientPort = 0; - try { - result.stage = "build"; - run(process.env.ComSpec || "cmd.exe", ["/d", "/s", "/c", "npm run build:client"]); - run(powershell(), [ - "-NoProfile", "-ExecutionPolicy", "Bypass", "-File", - join(ROOT, "native", "capture", "windows", "build.ps1"), - "-OutputDirectory", BUILD_ROOT, - ]); - run(go, ["build", "-trimpath", "-o", clientBinary, "./cmd/screener-client"], - ROOT); - - result.stage = "livekit"; - livekit = spawn(livekitPath, [ - "--bind", "127.0.0.1", - "--node-ip", "127.0.0.1", - "--port", String(livekitPort), - "--udp-port", String(livekitUDPPort), - "--rtc.tcp_port", String(livekitTCPPort), - "--rtc.enable_loopback_candidate", - ], { - stdio: "pipe", - windowsHide: true, - env: { ...process.env, LIVEKIT_KEYS: `${apiKey}: ${apiSecret}` }, - }); - livekit.stdout.resume(); - livekit.stderr.resume(); - await waitForHTTP(`http://127.0.0.1:${livekitPort}`, 10_000); - - result.stage = "vite"; - vite = await createViteServer({ - root: ROOT, - appType: "custom", - logLevel: "silent", - server: { host: "127.0.0.1", port: vitePort, strictPort: true }, - plugins: [{ - name: "native-sfu-gate-page", - configureServer(server) { - server.middlewares.use((request, response, next) => { - if (request.url?.split("?", 1)[0] !== "/native-sfu-gate") { - next(); - return; - } - response.statusCode = 200; - response.setHeader("Content-Type", "text/html; charset=utf-8"); - response.end(""); - }); - }, - }], - }); - await vite.listen(); - - result.stage = "source"; - source = await sourceServer(sourcePort); - ({ child: sourceChrome, cdp: sourceCdp } = await startSourceBrowser( - chromePath, - sourceProfile, - sourceDebugPort, - sourcePort, - )); - await waitForCaptureWindow(captureBinary); - - result.stage = "client"; - client = spawn(clientBinary, [ - "--site", origin, - "--capture-process", captureBinary, - "--config", configPath, - ], { - stdio: "pipe", - windowsHide: true, - env: { ...process.env, SCREENER_CLIENT_GATE_NO_BROWSER: "true" }, - }); - client.stderr.resume(); - clientPort = (await readClientEndpoint(client, { - ignoreNonEndpointLines: true, - })).port; - - result.stage = "browser"; - chrome = launchChrome(chromePath, debugPort, profile, [ - "--no-first-run", - "--no-default-browser-check", - "--disable-extensions", - "--disable-logging", - "--disable-background-timer-throttling", - "--disable-backgrounding-occluded-windows", - "--disable-renderer-backgrounding", - "--disable-features=WebRtcHideLocalIpsWithMdns", - ]); - chrome.stdout.resume(); - chrome.stderr.resume(); - const version = await waitForVersion(debugPort, chrome); - cdp = await CdpConnection.connect(version.webSocketDebuggerUrl, Date.now() + 10_000); - hostPage = await createPage(cdp, `${origin}/native-sfu-gate`, undefined, true); - viewerPage = await createPage(cdp, `${origin}/native-sfu-gate`, undefined, true); - const hostToken = await issueToken(apiKey, apiSecret, room, "host", true); - const viewerToken = await issueToken(apiKey, apiSecret, room, "viewer", false); - - result.stage = "publish"; - const hostResult = await evaluate<{ audio?: boolean; error?: string }>( - cdp, - hostPage, - `(async () => { try { - const gate = await import('/scripts/client-native-sfu-page.ts'); - return await gate.startNativeSfuHost(${JSON.stringify({ - url: livekitURL, - token: hostToken, - sourceTitle: SOURCE_TITLE, - })}); - } catch (error) { - return { error: String(error?.stack || error?.message || error) }; - } })()`, - Date.now() + 30_000, - ); - if (hostResult.error) throw new Error(hostResult.error); - result.hostPublished = true; - - result.stage = "subscribe"; - const viewer = await evaluate<{ - frames?: number; - width?: number; - height?: number; - error?: string; - }>( - cdp, - viewerPage, - `(async () => { try { - const gate = await import('/scripts/client-native-sfu-page.ts'); - return await gate.startNativeSfuViewer(${JSON.stringify({ - url: livekitURL, - token: viewerToken, - })}); - } catch (error) { - return { error: String(error?.stack || error?.message || error) }; - } })()`, - Date.now() + 30_000, - ); - if (viewer.error) throw new Error(viewer.error); - result.viewerFrames = viewer.frames ?? 0; - result.viewerWidth = viewer.width ?? 0; - result.viewerHeight = viewer.height ?? 0; - result.stage = "profile-update"; - const updated = await evaluate( - cdp, - hostPage, - `(async () => { - const gate = await import('/scripts/client-native-sfu-page.ts'); - return await gate.updateNativeSfuHost({ - resolution: '480p', - maxFramerate: 15, - maxBitrate: 2000000, - degradationPreference: 'maintain-resolution', - screenAudioQuality: 'saver', - }); - })()`, - Date.now() + 20_000, - ); - const updatedViewer = await evaluate<{ - frames: number; - width: number; - height: number; - }>( - cdp, - viewerPage, - `import('/scripts/client-native-sfu-page.ts').then((gate) => - gate.waitForNativeSfuViewer(854, 480))`, - Date.now() + 25_000, - ); - result.profileUpdated = updated && updatedViewer.frames >= 15; - result.viewerFrames = updatedViewer.frames; - result.viewerWidth = updatedViewer.width; - result.viewerHeight = updatedViewer.height; - } catch (error) { - result.error = error instanceof Error ? error.message : String(error); - } finally { - if (cdp && viewerPage) { - await evaluate( - cdp, - viewerPage, - "import('/scripts/client-native-sfu-page.ts').then((gate) => gate.stopNativeSfuGate())", - Date.now() + 5_000, - ).catch(() => undefined); - } - if (cdp && hostPage) { - await evaluate( - cdp, - hostPage, - "import('/scripts/client-native-sfu-page.ts').then((gate) => gate.stopNativeSfuGate())", - Date.now() + 5_000, - ).catch(() => undefined); - } - await vite?.close().catch(() => undefined); - if (sourceChrome && sourceCdp) { - await closeSourceBrowser(sourceChrome, sourceCdp).catch(() => false); - sourceChrome = null; - sourceCdp = null; - } - await source?.close().catch(() => undefined); - const livekitStopped = await stopProcess(livekit); - const cleanup = await cleanupRun({ - cdp, - native: client, - chrome, - server: null, - profile, - ports: [ - vitePort, - debugPort, - livekitPort, - livekitTCPPort, - ...(clientPort ? [clientPort] : []), - ], - }); - const sourceCleanup = await cleanupRun({ - cdp: null, - native: sourceChrome, - chrome: null, - server: null, - profile: sourceProfile, - ports: [sourcePort, sourceDebugPort], - }); - result.cleanup = livekitStopped && cleanup.browserExited && - cleanup.nativeExited && cleanup.portsClosed && cleanup.profileRemoved && - sourceCleanup.profileRemoved && sourceCleanup.portsClosed; - } - result.passed = result.error === null && result.hostPublished && - result.profileUpdated && result.viewerFrames >= 15 && - result.viewerWidth === 854 && result.viewerHeight === 480 && result.cleanup; - process.stdout.write(`${JSON.stringify(result)}\n`); - if (!result.passed) process.exitCode = 1; -} - -await main(); diff --git a/scripts/client-native-sfu-page.ts b/scripts/client-native-sfu-page.ts deleted file mode 100644 index ee3c67e..0000000 --- a/scripts/client-native-sfu-page.ts +++ /dev/null @@ -1,186 +0,0 @@ -import { - DEFAULT_QUALITY_SETTINGS, - type QualitySettings, -} from "../src/shared/protocol"; -import { NativeClient } from "../src/client/native/client"; -import { NativeMediaBridge } from "../src/client/native/media-bridge"; -import { SfuPublisher } from "../src/client/sfu/publisher"; - -let hostResources: { - client: NativeClient; - bridge: NativeMediaBridge; - publisher: SfuPublisher; - shareId: string; -} | null = null; -let viewerRoom: import("livekit-client").Room | null = null; -let viewerVideo: HTMLVideoElement | null = null; - -export async function startNativeSfuHost(input: { - url: string; - token: string; - sourceTitle: string; -}): Promise<{ audio: boolean }> { - if (hostResources) throw new Error("native SFU Host is already active"); - const client = await NativeClient.connect(); - if (!client) throw new Error("native Client is unavailable"); - const shareId = "native_sfu_share_123456"; - let bridge: NativeMediaBridge | null = null; - let publisher: SfuPublisher | null = null; - try { - const adapters = await client.captureOptions(); - const adapter = adapters.find((candidate) => candidate.hardwareH264.length > 0); - const encoder = adapter?.hardwareH264[0]; - const sources = await client.sources(); - const target = sources.find((candidate) => - candidate.kind === "window" && - candidate.title.includes(input.sourceTitle) - ); - if (!adapter || !encoder || !target) { - throw new Error("native capture path is unavailable"); - } - const started = await client.startShare({ - codec: "h264", - shareId, - source: target, - audio: true, - adapterIndex: adapter.index, - encoderIndex: encoder.index, - edgeCapacity: 3, - profile: DEFAULT_QUALITY_SETTINGS, - }); - bridge = new NativeMediaBridge( - shareId, - client, - () => undefined, - started.audio, - ); - const stream = await bridge.start(); - publisher = new SfuPublisher(); - if (!(await publisher.connect({ url: input.url, token: input.token }))) { - throw new Error("native SFU publisher did not connect"); - } - if (!(await publisher.activate(stream, DEFAULT_QUALITY_SETTINGS, "h264"))) { - throw new Error("native SFU publisher did not activate"); - } - hostResources = { client, bridge, publisher, shareId }; - return { audio: stream.getAudioTracks().length > 0 }; - } catch (error) { - await publisher?.disconnect().catch(() => undefined); - bridge?.dispose(); - await client.stopShare(shareId).catch(() => undefined); - client.close(); - throw error; - } -} - -export async function startNativeSfuViewer(input: { - url: string; - token: string; -}): Promise<{ frames: number; width: number; height: number }> { - if (viewerRoom) throw new Error("native SFU Viewer is already active"); - const sdk = await import("livekit-client"); - const room = new sdk.Room({ - adaptiveStream: false, - disconnectOnPageLeave: false, - }); - viewerRoom = room; - const result = new Promise<{ frames: number; width: number; height: number }>( - (resolve, reject) => { - const timer = window.setTimeout( - () => reject(new Error("native SFU Viewer timed out")), - 20_000, - ); - room.on(sdk.RoomEvent.TrackSubscribed, (track, publication) => { - if (track.kind !== sdk.Track.Kind.Video) return; - publication.setVideoQuality(sdk.VideoQuality.HIGH); - const video = track.attach() as HTMLVideoElement; - viewerVideo = video; - video.muted = true; - video.autoplay = true; - document.body.append(video); - void video.play().catch(() => undefined); - let frames = 0; - const next = () => { - video.requestVideoFrameCallback(() => { - if (video.videoWidth === 1920 && video.videoHeight === 1080) { - frames += 1; - } else { - frames = 0; - } - if (frames < 30) { - next(); - return; - } - window.clearTimeout(timer); - resolve({ - frames, - width: video.videoWidth, - height: video.videoHeight, - }); - }); - }; - next(); - }); - }, - ); - try { - await room.connect(input.url, input.token); - return await result; - } catch (error) { - await room.disconnect(false).catch(() => undefined); - viewerRoom = null; - throw error; - } -} - -export async function updateNativeSfuHost( - profile: QualitySettings, -): Promise { - const host = hostResources; - if (!host) throw new Error("native SFU Host is unavailable"); - await host.client.updateShare(host.shareId, profile); - return await host.publisher.updateProfile(profile); -} - -export async function waitForNativeSfuViewer( - width: number, - height: number, -): Promise<{ frames: number; width: number; height: number }> { - const video = viewerVideo; - if (!video) throw new Error("native SFU Viewer is unavailable"); - return await new Promise((resolve, reject) => { - let frames = 0; - const timer = window.setTimeout( - () => reject(new Error("native SFU Viewer profile timed out")), - 20_000, - ); - const next = () => { - video.requestVideoFrameCallback(() => { - frames = video.videoWidth === width && video.videoHeight === height - ? frames + 1 - : 0; - if (frames < 15) { - next(); - return; - } - window.clearTimeout(timer); - resolve({ frames, width: video.videoWidth, height: video.videoHeight }); - }); - }; - next(); - }); -} - -export async function stopNativeSfuGate(): Promise { - const viewer = viewerRoom; - viewerRoom = null; - viewerVideo = null; - if (viewer) await viewer.disconnect(false).catch(() => undefined); - const host = hostResources; - hostResources = null; - if (!host) return; - await host.publisher.disconnect().catch(() => undefined); - host.bridge.dispose(); - await host.client.stopShare(host.shareId).catch(() => undefined); - host.client.close(); -} diff --git a/scripts/client-package-targets.mjs b/scripts/client-package-targets.mjs index 24044a8..6976394 100644 --- a/scripts/client-package-targets.mjs +++ b/scripts/client-package-targets.mjs @@ -7,6 +7,7 @@ export const CLIENT_PACKAGE_TARGETS = [ nodeArch: "x64", goos: "windows", goarch: "amd64", + cgo: false, clientName: "screener-client.exe", tunnelName: "cloudflared.exe", captureName: "screener-client-capture.exe", @@ -20,6 +21,7 @@ export const CLIENT_PACKAGE_TARGETS = [ nodeArch: "x64", goos: "linux", goarch: "amd64", + cgo: false, clientName: "screener-client", tunnelName: "cloudflared", captureName: "screener-client-capture", @@ -33,6 +35,7 @@ export const CLIENT_PACKAGE_TARGETS = [ nodeArch: "arm64", goos: "darwin", goarch: "arm64", + cgo: true, clientName: "screener-client", tunnelName: "cloudflared", captureName: "screener-client-capture", @@ -45,3 +48,12 @@ export const CLIENT_PACKAGE_TARGETS = [ export function clientPackageTarget(id) { return CLIENT_PACKAGE_TARGETS.find((target) => target.id === id) ?? null; } + +export function clientGoEnvironment(target) { + return { + ...process.env, + GOOS: target.goos, + GOARCH: target.goarch, + CGO_ENABLED: target.cgo ? "1" : "0", + }; +} diff --git a/scripts/client-package-targets.test.mjs b/scripts/client-package-targets.test.mjs new file mode 100644 index 0000000..89bb34b --- /dev/null +++ b/scripts/client-package-targets.test.mjs @@ -0,0 +1,25 @@ +import assert from "node:assert/strict"; +import { test } from "node:test"; +import { CLIENT_PACKAGE_TARGETS, clientGoEnvironment } from "./client-package-targets.mjs"; + +test("build and license discovery use the same explicit target cgo policy", () => { + const previous = process.env.CGO_ENABLED; + try { + process.env.CGO_ENABLED = "unexpected-inherited-value"; + const settings = CLIENT_PACKAGE_TARGETS.map((target) => { + const env = clientGoEnvironment(target); + const pathKey = Object.keys(process.env).find((key) => key.toLowerCase() === "path"); + if (pathKey) assert.equal(env[pathKey], process.env[pathKey]); + assert.equal(process.env.CGO_ENABLED, "unexpected-inherited-value"); + return [target.id, env.GOOS, env.GOARCH, env.CGO_ENABLED]; + }); + assert.deepEqual(settings, [ + ["windows-amd64", "windows", "amd64", "0"], + ["linux-amd64", "linux", "amd64", "0"], + ["darwin-arm64", "darwin", "arm64", "1"], + ]); + } finally { + if (previous === undefined) delete process.env.CGO_ENABLED; + else process.env.CGO_ENABLED = previous; + } +}); diff --git a/scripts/embedded-sfu-gate.ts b/scripts/embedded-sfu-gate.ts new file mode 100644 index 0000000..4cf212e --- /dev/null +++ b/scripts/embedded-sfu-gate.ts @@ -0,0 +1,303 @@ +import { spawn, spawnSync, type ChildProcessWithoutNullStreams } from "node:child_process"; +import { createSocket } from "node:dgram"; +import { mkdir, mkdtemp } from "node:fs/promises"; +import { tmpdir } from "node:os"; +import { join, resolve } from "node:path"; +import { createServer as createViteServer, type ViteDevServer } from "vite"; +import type { CreateRoomResponse } from "../src/shared/protocol"; +import { + CdpConnection, cleanupRun, createPage, evaluate, fetchJsonBefore, launchChrome, + reservePort, waitForSample, waitForVersion, withDeadline, type PageHandle, +} from "./browser-gate-harness"; +import type { snapshot } from "./embedded-sfu-page"; +import { readClientEndpoint } from "./client-gate-endpoint"; +import { SOURCE_TITLE, sourceServer, startSourceBrowser, waitForCaptureWindow } from "./client-native-host-gate"; + +const ROOT = resolve(import.meta.dirname, ".."); +const BUILD_ROOT = join(ROOT, "build", "embedded-media"); +type Snapshot = ReturnType; + +const AUDIO_PROBE = `(() => { + const peers = [], BrowserPeer = RTCPeerConnection; + window.RTCPeerConnection = class extends BrowserPeer { + constructor(...args) { super(...args); peers.push(this); } + }; + window.__screenerGateAudioEnergy = async () => { + const trackId = document.querySelector('video')?.srcObject?.getAudioTracks()[0]?.id; + const diagnostic = window.__screenerGateAudioStats = { hasAudioTrack: !!trackId, peerCount: peers.length, inbound: [] }; + if (!trackId) return 0; + for (const peer of peers) { + if (peer.connectionState !== 'connected') continue; + const report = await peer.getStats(); + for (const row of report.values()) { + if (row.type === 'inbound-rtp' && row.kind === 'audio') diagnostic.inbound.push({ + trackMatches: row.trackIdentifier === trackId, energy: row.totalAudioEnergy ?? null, + sampleDuration: row.totalSamplesDuration ?? null, receivedSamples: row.totalSamplesReceived ?? null, + }); + if (row.type === 'inbound-rtp' && row.kind === 'audio' && row.trackIdentifier === trackId && + Number.isFinite(row.totalAudioEnergy) && row.totalAudioEnergy > 0) return row.totalAudioEnergy; + } + } + return 0; + }; +})()`; + +async function udpPort(requested = 0): Promise { + const socket = createSocket("udp4"); + return await new Promise((resolvePort, reject) => { + socket.once("error", reject); + socket.bind(requested, "127.0.0.1", () => { + const port = socket.address().port; + socket.close(() => resolvePort(port)); + }); + }); +} + +async function main(): Promise { + if (process.env.SCREENER_EMBEDDED_SFU_GATE !== "true") { + throw new Error("SCREENER_EMBEDDED_SFU_GATE=true is required; this is an explicit local Browser media gate"); + } + const chromePath = process.env.CHROME_PATH?.trim(); + if (!chromePath) throw new Error("CHROME_PATH is required"); + const nativeArm = process.env.SCREENER_EMBEDDED_SFU_NATIVE === "true"; + const twoRooms = process.env.SCREENER_EMBEDDED_SFU_TWO_ROOMS === "true"; + if (twoRooms && !nativeArm) throw new Error("Two-room arm requires Native publication"); + const codec = process.env.SCREENER_EMBEDDED_SFU_CODEC?.trim() || "vp8"; + if (codec !== "vp8" && codec !== "h264") throw new Error("SFU gate codec must be vp8 or h264"); + const captureBinary = process.env.SCREENER_CAPTURE_PROCESS?.trim(); + if (nativeArm && (process.platform !== "win32" || !captureBinary)) { + throw new Error("Native arm requires Windows and SCREENER_CAPTURE_PROCESS"); + } + await mkdir(BUILD_ROOT, { recursive: true }); + const binary = join(BUILD_ROOT, `screener-server${process.platform === "win32" ? ".exe" : ""}`); + const clientBinary = join(BUILD_ROOT, "screener-client.exe"); + for (const command of nativeArm ? ["screener-server", "screener-client"] : ["screener-server"]) { + const build = spawnSync(process.env.SCREENER_GO?.trim() || "go", [ + "build", "-p", "1", "-trimpath", "-o", command === "screener-server" ? binary : clientBinary, `./cmd/${command}`, + ], { cwd: ROOT, env: { ...process.env, GOMAXPROCS: "2" }, encoding: "utf8", windowsHide: true, timeout: 120_000 }); + if (build.error || build.status !== 0) throw build.error ?? new Error(build.stderr || `${command} build failed`); + } + const [serverPort, vitePort, debugPort, mediaPort] = await Promise.all([ + reservePort(), reservePort(), reservePort(), udpPort(), + ]); + const origin = `http://127.0.0.1:${vitePort}`; + const backend = `http://127.0.0.1:${serverPort}`; + const profile = await mkdtemp(join(tmpdir(), "screener-client-media-")); + const sourceProfile = nativeArm ? await mkdtemp(join(tmpdir(), "screener-client-media-")) : null; + const [sourcePort, sourceDebugPort] = nativeArm ? [await reservePort(), await reservePort()] : [0, 0]; + const result = { passed: false, arm: `${nativeArm ? "native" : "browser"}-${codec}-simulcast${twoRooms ? "-two-rooms" : ""}`, stage: "start", + startedAt: new Date().toISOString(), finishedAt: "", processes: [] as Array<{ role: string; pid: number | null }>, + high: null as { frames: number; width: number; height: number } | null, + low: null as { frames: number; width: number; height: number } | null, + audioEnergy: { high: 0, low: 0 }, + audioDiagnostics: null as unknown, + secondRoom: twoRooms ? { connected: false, framesAfterFirstStopped: 0, audioBefore: 0, audioAfter: 0, stopped: false } : null, + host: null as Snapshot | null, viewer: null as Snapshot | null, + publicationRetired: false, nativeShareStopped: !nativeArm, udpReleased: false, cleanup: false, error: null as string | null }; + let server: ChildProcessWithoutNullStreams | null = null; + let chrome: ChildProcessWithoutNullStreams | null = null; + let vite: ViteDevServer | null = null; + let cdp: CdpConnection | null = null; + let host: PageHandle | null = null; + let viewer: PageHandle | null = null; + let secondHost: PageHandle | null = null; + let secondViewer: PageHandle | null = null; + let serverError = ""; + let client: ChildProcessWithoutNullStreams | null = null; + let clientPort = 0; + let sourceChrome: ChildProcessWithoutNullStreams | null = null; + let sourceCdp: CdpConnection | null = null; + let sourceHTTP: Awaited> | null = null; + const call = async (page: PageHandle, expression: string, timeout = 25_000): Promise => { + const value = await evaluate<{ value?: T; error?: string }>(cdp!, page, + `import('/scripts/embedded-sfu-page.ts').then(async (gate) => { try { return { value: await (${expression}) }; } catch (error) { return { error: String(error?.message || error) }; } })`, + Date.now() + timeout); + if (value.error) throw new Error(value.error); + return value.value as T; + }; + const until = async (page: PageHandle, accept: (value: Snapshot) => boolean) => { + let last: Snapshot | null = null; + try { + await waitForSample(() => call(page, "gate.snapshot()", 3_000), + (value) => { last = value; return !!value.error || accept(value); }, 25_000); + } catch { throw new Error(`Browser stage timed out: ${JSON.stringify(last)}`); } + if (last!.error) throw new Error(last!.error); + return last!; + }; + const processStarted = (role: string, child: ChildProcessWithoutNullStreams) => { + const entry = { role, pid: child.pid ?? null }; + result.processes.push(entry); + process.stderr.write(`${JSON.stringify({ ...entry, startedAt: new Date().toISOString() })}\n`); + }; + try { + server = spawn(binary, [], { cwd: ROOT, stdio: "pipe", windowsHide: true, + env: { PATH: process.env.PATH, SystemRoot: process.env.SystemRoot, + SCREENER_ENV: "development", LISTEN_HOST: "127.0.0.1", PORT: String(serverPort), + PUBLIC_BASE_URL: origin, ALLOWED_ORIGINS: origin, + SFU_LISTEN_HOST: "127.0.0.1", SFU_UDP_PORT: String(mediaPort), SFU_PUBLIC_IP: "127.0.0.1" }, + }); + processStarted("server", server); + server.stdout.resume(); + server.stderr.on("data", (data: Buffer) => { serverError = (serverError + data.toString()).slice(-2000); }); + await waitForSample((deadline) => fetchJsonBefore<{ status: string }>(`${backend}/healthz`, deadline), + () => true, 15_000); + vite = await createViteServer({ root: ROOT, configFile: false, appType: "custom", logLevel: "silent", + server: { host: "127.0.0.1", port: vitePort, strictPort: true, watch: null, + proxy: { "/api": { target: backend }, "/signal": { target: backend, ws: true } } }, + plugins: [{ name: "embedded-sfu-gate", configureServer(instance) { + instance.middlewares.use((request, response, next) => { + if (request.url !== "/embedded-sfu-gate") return next(); + response.setHeader("Content-Type", "text/html; charset=utf-8"); + response.end("Embedded SFU acceptance"); + }); + } }], + }); + await vite.listen(); + if (nativeArm) { + result.stage = "native-client-start"; + client = spawn(clientBinary, ["--site", origin, "--capture-process", captureBinary!, + "--config", join(profile, "client.json")], { cwd: ROOT, stdio: "pipe", windowsHide: true, + env: { ...process.env, GOMAXPROCS: "2", SCREENER_CLIENT_GATE_NO_BROWSER: "true" } }); + processStarted("client", client); + client.stderr.resume(); + clientPort = (await readClientEndpoint(client, { timeoutMs: 15_000, ignoreNonEndpointLines: true })).port; + client.stdout.resume(); + result.stage = "native-source-window"; + sourceHTTP = await sourceServer(sourcePort); + ({ child: sourceChrome, cdp: sourceCdp } = await startSourceBrowser(chromePath, sourceProfile!, sourceDebugPort, sourcePort)); + processStarted("source-browser", sourceChrome); + await waitForCaptureWindow(captureBinary!); + const targets = await sourceCdp.call<{ targetInfos: Array<{ targetId: string; url: string }> }>( + "Target.getTargets", {}, undefined, Date.now() + 5_000); + const target = targets.targetInfos.find((entry) => entry.url === `http://127.0.0.1:${sourcePort}/`); + if (!target) throw new Error("Gate source page is unavailable"); + const attached = await sourceCdp.call<{ sessionId: string }>("Target.attachToTarget", + { targetId: target.targetId, flatten: true }, undefined, Date.now() + 5_000); + const audioState = await evaluate(sourceCdp, attached, `(async () => { + const audio = new AudioContext(), tone = audio.createOscillator(), gain = audio.createGain(); + gain.gain.value = 0.01; tone.connect(gain).connect(audio.destination); tone.start(); + window.__gateAudio = audio; await audio.resume(); return audio.state; + })()`, Date.now() + 5_000); + if (audioState !== "running") throw new Error("Native source tone did not start"); + } + chrome = launchChrome(chromePath, debugPort, profile, [ + "--headless=new", "--disable-gpu", "--no-first-run", "--no-default-browser-check", + "--disable-extensions", "--disable-background-networking", "--autoplay-policy=no-user-gesture-required", + "--disable-background-timer-throttling", "--disable-backgrounding-occluded-windows", + "--disable-renderer-backgrounding", "--disable-features=WebRtcHideLocalIpsWithMdns", + ]); + processStarted("decoder-browser", chrome); + chrome.stdout.resume(); + chrome.stderr.resume(); + const version = await waitForVersion(debugPort, chrome); + cdp = await CdpConnection.connect(version.webSocketDebuggerUrl, Date.now() + 10_000); + if (nativeArm) await cdp.call("Browser.setPermission", { + permission: { name: "local-network-access" }, setting: "granted", origin, + }, undefined, Date.now() + 5_000); + result.stage = "gate-page"; + const gatePage = await fetch(`${origin}/embedded-sfu-gate`, { signal: AbortSignal.timeout(5_000) }); + if (!gatePage.ok || !(await gatePage.text()).includes("Embedded SFU acceptance")) { + throw new Error("Local SFU gate page is unavailable"); + } + host = await createPage(cdp, `${origin}/embedded-sfu-gate`); + viewer = await createPage(cdp, `${origin}/embedded-sfu-gate`, AUDIO_PROBE); + result.stage = "authenticate"; + const room = await call(host, nativeArm + ? `gate.startHost(${JSON.stringify({ title: SOURCE_TITLE, port: clientPort })}, ${JSON.stringify(codec)})` + : `gate.startHost(undefined, ${JSON.stringify(codec)})`); + await until(host, (value) => value.authenticated); + await call(viewer, `gate.startViewer(${JSON.stringify(room)})`); + result.stage = "simulcast-first-frame"; + result.viewer = await until(viewer, (value) => value.committed && value.audioKbps > 0); + if (twoRooms) { + result.stage = "second-native-room"; + secondHost = await createPage(cdp, `${origin}/embedded-sfu-gate`); + secondViewer = await createPage(cdp, `${origin}/embedded-sfu-gate`, AUDIO_PROBE); + const secondRoom = await call(secondHost, + `gate.startHost(${JSON.stringify({ title: SOURCE_TITLE, port: clientPort })}, ${JSON.stringify(codec)})`); + if (secondRoom.roomId === room.roomId) throw new Error("Native sessions did not create independent rooms"); + await until(secondHost, (value) => value.authenticated); + await call(secondViewer, `gate.startViewer(${JSON.stringify(secondRoom)})`); + await until(secondViewer, (value) => value.committed && value.audioKbps > 0); + await call(secondViewer, "gate.waitForFrames(1280, 720)"); + await waitForSample(() => call(secondViewer!, "gate.audioEnergy()", 3_000), + (energy) => { result.secondRoom!.audioBefore = energy; return energy > 0; }, 5_000); + const second = await call(secondHost, "gate.snapshot()"); + result.secondRoom!.connected = second.published && second.simulcast && second.nativeShareStarted; + } + result.stage = "simulcast-first-frame"; + await cdp.call("Page.bringToFront", {}, viewer.sessionId, Date.now() + 3_000); + result.host = await call(host, "gate.snapshot()"); + result.high = await call(viewer, nativeArm ? "gate.waitForFrames(1280, 720, 300)" : "gate.waitForFrames(1280, 720)", nativeArm ? 32_000 : 25_000); + result.stage = "decoded-audio-energy"; + await waitForSample(() => call(viewer!, "gate.audioEnergy()", 3_000), + (energy) => { result.audioEnergy.high = energy; return energy > 0; }, 5_000); + result.host = await call(host, "gate.snapshot()"); + result.stage = "live-profile"; + if (!await call(host, "gate.lowerProfile()")) { + result.host = await call(host, "gate.snapshot()"); + throw new Error("Live profile update failed"); + } + result.low = await call(viewer, "gate.waitForFrames(854, 480)"); + await waitForSample(() => call(viewer!, "gate.audioEnergy()", 3_000), + (energy) => { result.audioEnergy.low = energy; return energy > result.audioEnergy.high; }, 5_000); + result.stage = "publication-retirement"; + await call(host, "gate.stopSharing()"); + await until(host, (value) => value.publicationRetired); + await until(viewer, (value) => value.sharingStopped); + result.publicationRetired = true; + if (secondHost && secondViewer) { + result.stage = "second-room-isolation"; + if (!await call(host, "gate.stop()")) throw new Error("First Native session did not stop"); + if (!(await call(host, "gate.snapshot()")).nativeShareStopped) throw new Error("First Native capture did not stop"); + result.secondRoom!.audioBefore = await call(secondViewer, "gate.audioEnergy()"); + await cdp.call("Page.bringToFront", {}, secondViewer.sessionId, Date.now() + 3_000); + const continued = await call<{ frames: number }>(secondViewer, "gate.waitForFrames(1280, 720, 30)"); + result.secondRoom!.framesAfterFirstStopped = continued.frames; + await waitForSample(() => call(secondViewer!, "gate.audioEnergy()", 3_000), + (energy) => { result.secondRoom!.audioAfter = energy; return energy > result.secondRoom!.audioBefore; }, 5_000); + if ((await call(secondHost, "gate.snapshot()")).error) throw new Error("First session retirement affected second Native Host"); + await call(secondHost, "gate.stopSharing()"); + await until(secondHost, (value) => value.publicationRetired); + await until(secondViewer, (value) => value.sharingStopped); + if (!await call(secondHost, "gate.stop()")) throw new Error("Second Native session did not stop"); + result.secondRoom!.stopped = (await call(secondHost, "gate.snapshot()")).nativeShareStopped; + } + } catch (error) { + result.error = error instanceof Error ? error.message : String(error); + if (viewer && cdp && result.stage === "decoded-audio-energy") result.audioDiagnostics = await evaluate( + cdp, viewer, "window.__screenerGateAudioStats ?? null", Date.now() + 3_000).catch(() => null); + if (server?.exitCode !== null && serverError) result.error += `; server: ${serverError}`; + } finally { + const pagesStopped = await Promise.all([host, viewer, secondHost, secondViewer].map((page) => + page && cdp ? call(page, "gate.stop()", 5_000).catch(() => false) : true)); + if (host && cdp && nativeArm) { + result.nativeShareStopped = await call(host, "gate.snapshot()", 3_000) + .then((value) => value.nativeShareStopped, () => false); + } + const viteClosed = vite + ? await withDeadline(() => vite!.close(), Date.now() + 5_000).then(() => true, () => false) + : true; + const cleanup = await cleanupRun({ cdp, native: client, chrome, server, profile, + ports: [serverPort, vitePort, debugPort, ...(clientPort ? [clientPort] : [])] }); + const sourceCleanup = sourceProfile ? await cleanupRun({ cdp: sourceCdp, native: null, + chrome: sourceChrome, server: sourceHTTP, profile: sourceProfile, ports: [sourcePort, sourceDebugPort] }) : null; + result.udpReleased = await withDeadline(() => udpPort(mediaPort), Date.now() + 3_000) + .then(() => true, () => false); + result.cleanup = viteClosed && result.udpReleased && pagesStopped.every(Boolean) && + Object.values(cleanup).every(Boolean) && (!sourceCleanup || Object.values(sourceCleanup).every(Boolean)); + } + result.finishedAt = new Date().toISOString(); + result.passed = !result.error && result.host?.published === true && result.host.simulcast && + result.viewer?.decoded === true && result.viewer.committed && result.viewer.peerFailures > 0 && + result.viewer.audioKbps > 0 && result.audioEnergy.high > 0 && result.audioEnergy.low > result.audioEnergy.high && + result.high !== null && result.low !== null && + result.publicationRetired && result.cleanup && result.nativeShareStopped && + (!result.secondRoom || result.secondRoom.connected && result.secondRoom.framesAfterFirstStopped >= 30 && + result.secondRoom.audioBefore > 0 && result.secondRoom.audioAfter > result.secondRoom.audioBefore && result.secondRoom.stopped) && + (!nativeArm || result.host.nativeShareStarted && result.host.nativeFramesPerSecond > 0 && result.host.nativeBitrateKbps > 0 && result.host.nativeEncodingCount === 2); + process.stdout.write(`${JSON.stringify(result)}\n`); + if (!result.passed) process.exitCode = 1; +} + +await main(); diff --git a/scripts/embedded-sfu-page.ts b/scripts/embedded-sfu-page.ts new file mode 100644 index 0000000..3b500c0 --- /dev/null +++ b/scripts/embedded-sfu-page.ts @@ -0,0 +1,315 @@ +import { + createRoomResponseSchema, + clientMessageSchema, + decodeServerMessage, + SIGNALING_PROTOCOL, + type ClientMessage, + type CreateRoomResponse, + type QualitySettings, + type ServerMessage, +} from "../src/shared/protocol"; +import { SfuPublisher } from "../src/client/sfu/publisher"; +import { SfuSubscriber } from "../src/client/sfu/subscriber"; +import type { SfuConnectionConfig } from "../src/client/sfu/peer"; +import { NativeClient } from "../src/client/native/client"; +import { NativeSfuPublisher } from "../src/client/native/native-sfu-publisher"; +import { defaultNativeCapturePath } from "../src/client/native/capture-selection"; +import type { HostPublisherTransport } from "../src/client/media/host-sfu-route"; + +const high: QualitySettings = { + resolution: "720p", maxFramerate: 15, maxBitrate: 2_000_000, + degradationPreference: "maintain-resolution", screenAudioQuality: "saver", +}; +const low: QualitySettings = { ...high, resolution: "480p" }; +const shareGeneration = crypto.randomUUID(); +let socket: WebSocket | null = null; +let publisher: HostPublisherTransport | null = null; +let native: NativeClient | null = null; +let nativeShareStarted = false; +let nativeShareStopped = false; +let nativeFramesPerSecond = 0; +let nativeBitrateKbps = 0; +let nativeEncodingCount = 0; +let subscriber: SfuSubscriber | null = null; +let configuration: SfuConnectionConfig | null = null; +let source: MediaStream | null = null; +let audio: AudioContext | null = null; +let canvas: HTMLCanvasElement | null = null; +let video: HTMLVideoElement | null = null; +let drawTimer: number | null = null; +let preparedRevision = 0; +let decoded = false; +let authenticated = false; +let committed = false; +let publicationRetired = false; +let sharingStopped = false; +let stopping = false; +let peerFailures = 0; +let published = false; +let simulcast = false; +let codec: "vp8" | "h264" = "vp8"; +let audioKbps = 0; +let audioCodec: string | null = null; +let error: string | null = null; +let messageTail = Promise.resolve(); + +function send(message: ClientMessage): boolean { + clientMessageSchema.parse(message); + if (socket?.readyState !== WebSocket.OPEN) return false; + if (message.type === "sfu-signal" && message.description?.type === "offer") { + simulcast = message.description.sdp.includes("a=simulcast:send q;h"); + } + socket.send(JSON.stringify(message)); + return true; +} + +function failed(reason: unknown): void { + error = reason instanceof Error ? reason.message : String(reason); +} + +async function accept(message: ServerMessage, host: boolean): Promise { + if (message.type === "error") throw new Error(`${message.code}: ${message.message}`); + if (message.type === "authenticated") authenticated = true; + if (message.type === "sharing-stopped") { + sharingStopped = true; + await subscriber?.disconnect(); + } + if (message.type === "route-update") { + if (message.phase === "prepare" && !host) { + if (message.candidate.transport === "direct") { + // Exercise the ordinary fallback operation without changing room policy. + peerFailures += 1; + send({ type: "route-failed", revision: message.revision, + phase: "prepare", connectionId: message.candidate.connectionId }); + } else preparedRevision = message.revision; + } + if (message.phase === "active") { + if (!host && message.assignment.upstream.kind === "sfu") { + if (!decoded) throw new Error("SFU committed before decoded-frame proof"); + committed = true; + } + if (configuration && message.assignment.sfuPublicationGeneration === configuration.publicationGeneration) { + configuration = { ...configuration, revision: message.revision }; + publisher?.updateConfig(configuration); + subscriber?.updateConfig(configuration); + } + } + } + if (message.type === "sfu-config") { + if (configuration) { + if (configuration.connectionId !== message.connectionId || + configuration.publicationGeneration !== message.publicationGeneration) { + throw new Error("Unexpected SFU replacement during bounded acceptance"); + } + configuration = message; + publisher?.updateConfig(message); + subscriber?.updateConfig(message); + return; + } + configuration = message; + if (host) { + const events = { send, onDisconnected: () => { + if (!stopping) failed("Publisher disconnected"); + } }; + publisher = native + ? new NativeSfuPublisher(native, shareGeneration, { iceServers: [] }, { ...events, + onStats: (metrics) => { + nativeFramesPerSecond = metrics?.framesPerSecond ?? 0; + nativeBitrateKbps = metrics?.bitrateKbps ?? 0; + nativeEncodingCount = metrics?.videoEncodingCount ?? 0; + }, + }) + : new SfuPublisher(events); + await publisher.connect(message); + published = await publisher.activate(source!, high, codec); + if (!published) throw new Error("Publisher did not activate"); + } else { + video = document.createElement("video"); + video.muted = false; + video.volume = 0.1; + video.autoplay = true; + video.playsInline = true; + video.style.cssText = "max-width:100%;width:640px"; + document.body.append(video); + subscriber = new SfuSubscriber({ + send, + onStream: (stream) => { + if (!video || video.srcObject === stream) return; + video.srcObject = stream; + if (stream) void video.play().catch(failed); + }, + onStats: (metrics) => { + audioKbps = metrics.audioBitrateKbps ?? 0; + audioCodec = metrics.audioCodec; + }, + onFirstDecodedFrame: () => { + decoded = true; + return send({ type: "route-ready", revision: preparedRevision, phase: "prepare" }); + }, + onDisconnected: () => { if (!sharingStopped) failed("Subscriber disconnected"); }, + }); + await subscriber.connect(message); + subscriber.armDecodedFrameProof(); + subscriber.activate(); + } + } + if (message.type === "sfu-signal") { + await publisher?.acceptSignal(message); + await subscriber?.acceptSignal(message); + } +} + +async function connect(room: CreateRoomResponse, host: boolean): Promise { + socket = new WebSocket(new URL("/signal", location.href).href.replace(/^http/, "ws")); + socket.addEventListener("message", ({ data }) => { + messageTail = messageTail.then(() => accept(decodeServerMessage(String(data)), host)).catch(failed); + }); + socket.addEventListener("error", () => failed("Room WebSocket failed")); + socket.addEventListener("close", (event) => { + if (host && stopping && event.code === 1000 && event.reason === "Sharing stopped") { + void publisher?.disconnect().then(() => { publicationRetired = true; }); + } else if (!stopping && !sharingStopped) failed(`Unexpected room socket close: ${event.code}`); + }); + await new Promise((resolve, reject) => { + const timer = setTimeout(() => reject(new Error("Room WebSocket timed out")), 10_000); + socket!.addEventListener("open", () => { clearTimeout(timer); resolve(); }, { once: true }); + }); + const common = { type: "authenticate", protocol: SIGNALING_PROTOCOL, + roomId: room.roomId, clientId: crypto.randomUUID() } as const; + send(host + ? { ...common, role: "host", token: room.hostToken, shareGeneration, + qualitySettings: high, routePolicy: { peerOnly: false, topologyOptimization: false, natPrediction: false } } + : { ...common, role: "viewer", viewerGrant: new URLSearchParams(new URL(room.inviteUrl).hash.slice(1)).get("v")! }); +} + +export async function startHost(nativeSource?: { title: string; port: number }, selectedCodec: "vp8" | "h264" = "vp8"): Promise { + codec = selectedCodec; + const response = await fetch("/api/rooms", { + method: "POST", headers: { "Content-Type": "application/json" }, + body: JSON.stringify({ codeEntryPolicy: "private" }), + }); + if (!response.ok) throw new Error(`Room creation failed: ${response.status}`); + const room = createRoomResponseSchema.parse(await response.json()); + if (nativeSource) { + native = await NativeClient.connect(); + if (!native || native.health.port !== nativeSource.port) throw new Error("Gate Client was not discovered"); + const target = (await native.sources()).find((candidate) => candidate.kind === "window" && candidate.title === nativeSource.title); + const path = defaultNativeCapturePath(await native.captureOptions(), codec, native.health.nativeMedia.softwareVP8); + if (!target || !path) throw new Error("Native source is unavailable"); + native.onEvent((event) => { + if (!stopping && event.type === "share-ended" && event.shareId === shareGeneration) failed("Native capture ended"); + }); + const started = await native.startShare({ shareId: shareGeneration, source: target, + codec, audio: true, profile: high, edgeCapacity: 2, ...path }); + nativeShareStarted = true; + if (started.codec !== codec || !started.audio) throw new Error("Requested Native codec and Opus capture did not start"); + await connect(room, true); + return room; + } + canvas = document.createElement("canvas"); + canvas.width = 1280; + canvas.height = 720; + canvas.style.cssText = "max-width:100%;width:640px"; + document.body.append(canvas); + const context = canvas.getContext("2d")!; + let frame = 0; + drawTimer = window.setInterval(() => { + context.fillStyle = "#167d8d"; + context.fillRect(0, 0, canvas!.width, canvas!.height); + context.fillStyle = "#f2d748"; + context.fillRect((frame++ * 17) % canvas!.width, 60, 100, 100); + context.fillStyle = "#ffffff"; + context.font = "48px sans-serif"; + context.fillText(String(frame), 40, 240); + }, 1000 / 15); + source = canvas.captureStream(15); + audio = new AudioContext(); + const tone = audio.createOscillator(); + const gain = audio.createGain(); + gain.gain.value = 0.05; + const destination = audio.createMediaStreamDestination(); + tone.connect(gain).connect(destination); + tone.start(); + await audio.resume(); + source.addTrack(destination.stream.getAudioTracks()[0]!); + await connect(room, true); + return room; +} + +export async function startViewer(room: CreateRoomResponse): Promise { + await connect(room, false); +} + +export function snapshot() { + return { authenticated, published, simulcast, decoded, committed, peerFailures, + audioKbps, audioCodec, publicationRetired, sharingStopped, nativeShareStarted, nativeShareStopped, + nativeFramesPerSecond, nativeBitrateKbps, nativeEncodingCount, + warning: publisher?.getQualityWarning?.() ?? null, + failureStage: publisher?.getFailureStage?.() ?? null, error }; +} + +export async function audioEnergy(): Promise { + const probe = (window as unknown as { __screenerGateAudioEnergy: () => Promise }).__screenerGateAudioEnergy; + return await probe(); +} + +export async function waitForFrames(width: number, height: number, minimumFrames = 15) { + if (!video) throw new Error("Viewer video is unavailable"); + const target = video; + return await new Promise<{ frames: number; width: number; height: number }>((resolve, reject) => { + let frames = 0; + let callback = 0; + const timer = setTimeout(() => { + target.cancelVideoFrameCallback(callback); + reject(new Error(`Viewer did not decode ${width}x${height}; current ${target.videoWidth}x${target.videoHeight}; ${frames} frame callbacks; ${document.visibilityState}; ${error ?? "no signaling error"}`)); + }, minimumFrames > 15 ? 30_000 : 20_000); + const next = () => { + callback = target.requestVideoFrameCallback(() => { + frames = target.videoWidth === width && target.videoHeight === height ? frames + 1 : 0; + if (frames < minimumFrames) return next(); + clearTimeout(timer); + resolve({ frames, width: target.videoWidth, height: target.videoHeight }); + }); + }; + next(); + }); +} + +export async function lowerProfile(): Promise { + if (!publisher) throw new Error("Publisher is unavailable"); + if (native) await native.updateShare(shareGeneration, low); + else { + if (!canvas) throw new Error("Browser source is unavailable"); + canvas.width = 854; + canvas.height = 480; + } + send({ type: "set-quality-settings", qualitySettings: low }); + return await publisher.updateProfile(low); +} + +export function stopSharing(): void { + stopping = true; + send({ type: "stop-sharing", shareGeneration }); +} + +export async function stop(): Promise { + stopping = true; + await publisher?.disconnect(); + await subscriber?.disconnect(); + if (native) { + if (nativeShareStarted) { + await native.stopShare(shareGeneration); + nativeShareStopped = true; + } + native.close(); + native = null; + } + if (drawTimer !== null) clearInterval(drawTimer); + source?.getTracks().forEach((track) => track.stop()); + await audio?.close(); + video?.remove(); + canvas?.remove(); + const closed = !source || source.getTracks().every((track) => track.readyState === "ended"); + socket?.close(); + return closed; +} diff --git a/scripts/encoded-group-decode.mjs b/scripts/encoded-group-decode.mjs new file mode 100644 index 0000000..473494d --- /dev/null +++ b/scripts/encoded-group-decode.mjs @@ -0,0 +1,83 @@ +import assert from "node:assert/strict"; +import { createHash } from "node:crypto"; +import { readFile, writeFile } from "node:fs/promises"; +import { createServer } from "node:http"; +import { resolve } from "node:path"; + +const customInput = process.env.SCREENER_ENCODED_OUTPUT; +const codec = process.env.SCREENER_ENCODED_CODEC ?? "vp8"; +assert.ok(["vp8", "avc1.42c033"].includes(codec), "Expected VP8 or constrained-baseline H264"); +const inputPath = customInput ? resolve(customInput) : new URL("../build/embedded-media/encoded-group.received.json", import.meta.url); +const input = await readFile(inputPath); +const streams = JSON.parse(input); +assert.equal(streams.length, 2); +for (const frames of streams) { + if (customInput) assert.ok(frames.length > 0 && frames.length <= 1200); + else assert.equal(frames.length, 40); +} +const expectedCounts = streams.map((frames) => frames.length); +const page = `Shared encoded output decode +
Decoding encoded input...
`; +const server = createServer(async (request, response) => { + if (request.url === "/frames") { + response.writeHead(200, { "Content-Type": "application/json" }).end(input); + } else if (request.url === "/result" && request.method === "POST") { + const body = []; + for await (const chunk of request) body.push(chunk); + const result = JSON.parse(Buffer.concat(body).toString()); + assert.equal(result.passed, true, result.error); + assert.deepEqual(result.results.map((frames) => frames.length), expectedCounts); + result.receivedSha256 = createHash("sha256").update(input).digest("hex"); + result.codec = codec; + result.scope = "Encoded input -> two Chrome WebCodecs decoders; input provenance is owned by its producing fixture, not proof of Browser WebRTC transport, jitter-buffer or hardware acceptance"; + result.sources = {}; + for (const path of ["go.mod", "go.sum", "internal/media/encoded/packetizer.go", + "internal/media/forwarding/source.go", "internal/media/forwarding/encoded_source.go", + "internal/media/forwarding/output.go", "internal/media/forwarding/transport.go", + "internal/client/mediaedge/engine.go", "internal/client/mediaedge/source.go", + "internal/client/mediaedge/edge.go", "internal/client/mediaedge/group_fixture_test.go", + "internal/client/mediaedge/relay_network_fixture_test.go", "internal/client/mediaedge/relay_derivation.go", + "native/capture/windows/encoded_group.fixture.cpp"]) { + result.sources[path] = createHash("sha256").update(await readFile(new URL(`../${path}`, import.meta.url))).digest("hex"); + } + const resultPath = customInput ? `${inputPath}.decode-result.json` : new URL("../build/embedded-media/encoded-group.decode-result.json", import.meta.url); + await writeFile(resultPath, `${JSON.stringify(result, null, 2)}\n`); + response.end("passed"); + console.log(`Passed: receivers decoded ${expectedCounts.join(" and ")} received frames`); + clearTimeout(deadline); + server.close(); + } else { + response.writeHead(200, { "Content-Type": "text/html; charset=utf-8" }).end(page); + } +}); +const deadline = setTimeout(() => { process.exitCode = 1; server.closeAllConnections(); server.close(); }, 60000); +server.listen(0, "127.0.0.1", () => console.log(`http://127.0.0.1:${server.address().port}`)); diff --git a/scripts/encoded-variants-browser.mjs b/scripts/encoded-variants-browser.mjs new file mode 100644 index 0000000..28db673 --- /dev/null +++ b/scripts/encoded-variants-browser.mjs @@ -0,0 +1,47 @@ +import { createServer } from "node:http"; +import { createHash } from "node:crypto"; +import { readFile, writeFile, mkdir } from "node:fs/promises"; +import { fileURLToPath } from "node:url"; + +const output = new URL("../build/embedded-media/browser-variants-result.json", import.meta.url); +const worker = await readFile(new URL("./encoded-variants-worker.js", import.meta.url)); +const page = `Encoded variants probe +
Ready
+`; +const server = createServer(async (request, response) => { + if (request.url === "/worker.js") { + response.writeHead(200, { "Content-Type": "text/javascript" }).end(worker); + } else if (request.url === "/result" && request.method === "POST") { + const chunks = []; + for await (const chunk of request) chunks.push(chunk); + const result = JSON.parse(Buffer.concat(chunks).toString()); + result.workerSha256 = createHash("sha256").update(worker).digest("hex"); + await mkdir(new URL("./", output), { recursive: true }); + await writeFile(output, `${JSON.stringify(result, null, 2)}\n`); + console.log(`Saved ${result.results.length} runs to ${fileURLToPath(output)}`); + response.end("saved"); + } else { + response.writeHead(200, { "Content-Type": "text/html; charset=utf-8" }).end(page); + } +}); +server.listen(0, "127.0.0.1", () => console.log(`http://127.0.0.1:${server.address().port}`)); +process.on("SIGINT", () => server.close()); diff --git a/scripts/encoded-variants-probe.mjs b/scripts/encoded-variants-probe.mjs new file mode 100644 index 0000000..be4e44f --- /dev/null +++ b/scripts/encoded-variants-probe.mjs @@ -0,0 +1,65 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { createHash } from "node:crypto"; +import { mkdirSync, readFileSync, writeFileSync } from "node:fs"; +import { cpus, platform, release } from "node:os"; +import { dirname, resolve } from "node:path"; + +const executable = resolve(process.argv[2] ?? "build/embedded-media/encoded-variants.probe.exe"); +const output = resolve("build/embedded-media/encoded-variants-result.json"); +const steady = ["high", "two-high", "high-low", "all", "mid-low", "low"]; +const traces = ["envelope", "exact-cold", "exact-warm", "warm-clock-ablation"]; +const results = []; +const report = { + completed: false, + scope: "CPU-only Native VP8 encode microbenchmark, not playback/network/H264 acceptance", + input: "32 synthetic NV12 frames per size; generation and scaling excluded", + profiles: [[480, 270, 312500], [960, 540, 1250000], [1920, 1080, 5000000]], + fps: 30, + threadsPerEncoder: 4, + demandTrace: ["H/H", "H/L", "L/L", "M/M", "H/H", "H/L", "H/H"], + secondsPerDemand: 2, + envelope: "maximum-demand active set; no LiveKit demand detection/debounce emulation", + warm: "retain at most three codec objects within the 14-second trace, stop inactive encoding", + clockAblation: "change only encoder PTS to a continuous active-frame clock; not a production proposal", + cpuMeasurement: "whole-window GetProcessTimes and QueryProcessCycleTime on Windows; cycles are not converted to milliseconds", + sources: Object.fromEntries(["native/capture/windows/encoded_variants.probe.cpp", "native/capture/windows/vp8_encoder.cpp"].map( + (path) => [path, createHash("sha256").update(readFileSync(path)).digest("hex")], + )), + host: { platform: platform(), release: release(), cpu: cpus()[0]?.model, logicalCpus: cpus().length }, + results, +}; +mkdirSync(dirname(output), { recursive: true }); + +for (let round = 0; round < 2; round++) { + for (const scene of ["still", "motion"]) { + const cases = [...steady, ...(scene === "motion" ? traces : [])]; + if (round) cases.reverse(); + for (const scenario of cases) { + const processResult = spawnSync(executable, [scenario, scene], { + encoding: "utf8", windowsHide: true, timeout: 45000, maxBuffer: 1024 * 1024, + }); + assert.equal(processResult.error, undefined, processResult.error?.message); + assert.equal(processResult.status, 0, processResult.stderr); + const result = JSON.parse(processResult.stdout); + assert.equal(result.scenario, scenario); + assert.equal(result.scene, scene); + if (traces.includes(scenario)) { + const expected = scenario === "envelope" ? [420, 360, 300, 0] : [180, 60, 300, 0]; + assert.deepEqual(result.summary.encodedFrames, expected); + assert.equal(result.transitions.length, 7); + assert.equal(result.summary.createCount, scenario === "exact-warm" || scenario === "warm-clock-ablation" ? 3 : 5); + } else { + const masks = { high: 4, "two-high": 12, "high-low": 5, all: 7, "mid-low": 3, low: 1 }; + assert.deepEqual(result.summary.encodedFrames, [0, 1, 2, 3].map((slot) => masks[scenario] & (1 << slot) ? 90 : 0)); + if (scenario === "two-high") assert.equal(result.summary.encodedBytes[2], result.summary.encodedBytes[3]); + } + results.push({ round: round + 1, ...result }); + writeFileSync(output, `${JSON.stringify(report, null, 2)}\n`); + console.log(`${round + 1} ${scene} ${scenario}: mean=${result.summary.workMeanMs}ms p95=${result.summary.workP95Ms}ms cpu=${result.summary.cpuCoresAt30Fps} cores`); + } + } +} +report.completed = true; +writeFileSync(output, `${JSON.stringify(report, null, 2)}\n`); +console.log(`Passed ${results.length} bounded runs; results: ${output}`); diff --git a/scripts/encoded-variants-worker.js b/scripts/encoded-variants-worker.js new file mode 100644 index 0000000..39d6ad2 --- /dev/null +++ b/scripts/encoded-variants-worker.js @@ -0,0 +1,221 @@ +// Isolated component experiment; no application transport, feedback or policy. +const profiles = [[480, 270, 312500], [960, 540, 1250000], [1920, 1080, 5000000]]; +const fps = 30; +const trace = [[2, 2], [2, 0], [0, 0], [1, 1], [2, 2], [2, 0], [2, 2]]; +const phaseFrames = 32; +const now = () => performance.now(); +const assert = (condition, message) => { if (!condition) throw Error(message); }; +const summarize = (values) => { + const sorted = [...values].sort((a, b) => a - b); + return { mean: values.reduce((a, b) => a + b, 0) / values.length, p95: sorted[Math.ceil(sorted.length * .95) - 1], max: sorted.at(-1) }; +}; + +function makeInput() { + return Array.from({ length: 32 }, (_, frame) => { + const data = new Uint8Array(1920 * 1080 * 3 / 2).fill(128); + for (let y = 0; y < 1080; y++) for (let x = 0; x < 1920; x++) { + const u = (x + frame * 4) % 128; + const v = y % 128; + data[y * 1920 + x] = 32 + ((u >> 3) ^ (v >> 3)) * 8 + ((u ^ v) & 31) * 2; + } + return data; + }); +} + +function config(codec, layer) { + const [width, height, bitrate] = profiles[layer]; + return { codec, width, height, bitrate, framerate: fps, latencyMode: "realtime", + bitrateMode: "constant", hardwareAcceleration: "prefer-software", + ...(codec.startsWith("avc") ? { avc: { format: "annexb" } } : {}) }; +} + +function encoder(codec, layer) { + let result; + let failure; + const value = new VideoEncoder({ output: (chunk, metadata) => { result = { chunk, metadata }; }, error: (error) => { failure = error; } }); + value.configure(config(codec, layer)); + return { + value, + async encode(frame, keyFrame) { + result = undefined; + value.encode(frame, { keyFrame }); + await value.flush(); + if (failure) throw failure; + assert(result?.chunk.timestamp === frame.timestamp, "Missing/out-of-order encoded frame"); + if (keyFrame) assert(result.chunk.type === "key", "Missing switch keyframe"); + return result; + }, + }; +} + +function decoder(codec) { + let complete; + let reject; + let layer = -1; + let failure; + const value = new VideoDecoder({ + output: (frame) => { const accept = complete; complete = undefined; if (accept) accept(frame); else frame.close(); }, + error: (error) => { failure = error; reject?.(error); }, + }); + return { + value, + async decode(packet, nextLayer) { + if (layer !== nextLayer) { + const [codedWidth, codedHeight] = profiles[nextLayer]; + value.configure({ codec, codedWidth, codedHeight, optimizeForLatency: true, hardwareAcceleration: "prefer-software" }); + layer = nextLayer; + } + if (failure) throw failure; + const promise = new Promise((resolve, rejectPromise) => { complete = resolve; reject = rejectPromise; }); + value.decode(packet); + const frame = await promise; + assert(frame.displayWidth === profiles[nextLayer][0] && frame.displayHeight === profiles[nextLayer][1], "Wrong decoded dimensions"); + return frame; + }, + }; +} + +function rawFrame(input, frame) { + return new VideoFrame(input[frame % input.length], { format: "NV12", codedWidth: 1920, codedHeight: 1080, + timestamp: Math.round(frame * 1e6 / fps), duration: Math.round(1e6 / fps) }); +} + +async function prepareIngress(codec, input) { + const source = encoder(codec, 2); + const packets = []; + try { + for (let i = 0; i < input.length; i++) { + const frame = rawFrame(input, i); + try { + const { chunk } = await source.encode(frame, i === 0); + const data = new Uint8Array(chunk.byteLength); + chunk.copyTo(data); + packets.push({ data, type: chunk.type }); + } finally { frame.close(); } + } + return packets; + } finally { source.value.close(); } +} + +async function run(codec, policy, role, input, ingress) { + const encoders = [null, null, null]; + const receivers = [decoder(codec), decoder(codec)]; + const inputDecoder = role === "relay" ? decoder(codec) : null; + let previous = [-1, -1]; + const encoded = [0, 0, 0]; + const generatedBytes = [0, 0, 0]; + const selectedBytes = [0, 0]; + const decoded = [0, 0]; + const switches = []; + const phases = []; + let creates = 0; + let ingressDecodes = 0; + const metrics = { management: [], input: [], encodeReady: [], allReceivers: [], processing: [] }; + const start = now(); + try { + for (let index = 0; index < trace.length * phaseFrames; index++) { + const phase = Math.floor(index / phaseFrames); + const demand = trace[phase]; + const tick = now(); + const demanded = new Set(demand); + const active = policy === "exact" ? demanded : new Set(Array.from({ length: Math.max(...demand) + 1 }, (_, i) => i)); + // A relay already has H. It never re-encodes H or starts lower encoders just for H/H. + if (role === "relay") { + active.delete(2); + if (!demand.some((i) => i < 2)) active.clear(); + } + const switched = demand.map((layer, child) => previous[child] !== layer); + const keyframes = new Set(demand.filter((_, child) => switched[child])); + for (let layer = 0; layer < 3; layer++) { + if (active.has(layer) && !encoders[layer]) { encoders[layer] = encoder(codec, layer); creates++; keyframes.add(layer); } + if (!active.has(layer) && encoders[layer]) { encoders[layer].value.close(); encoders[layer] = null; } + } + const afterManagement = now(); + let sourceFrame; + const outputs = new Map(); + const deliveries = []; + const deliver = async (layer, child, packetPromise) => { + const packet = await packetPromise; + assert(packet, "Missing demanded encoding"); + if (switched[child]) assert(packet.type === "key", "Switch did not start at a recovery frame"); + const frame = await receivers[child].decode(packet, layer); + assert(frame.timestamp === Math.round(index * 1e6 / fps), "Wrong decoded timestamp"); + frame.close(); + decoded[child]++; + selectedBytes[child] += packet.byteLength; + const delivered = now(); + if (switched[child] && index > 0) switches.push({ index, child, layer, ms: delivered - tick, bytes: packet.byteLength }); + return delivered; + }; + if (role === "relay") { + const sourcePacket = ingress[index % ingress.length]; + const packet = new EncodedVideoChunk({ ...sourcePacket, timestamp: Math.round(index * 1e6 / fps) }); + outputs.set(2, Promise.resolve(packet)); + demand.forEach((layer, child) => { if (layer === 2) deliveries[child] = deliver(layer, child, outputs.get(layer)); }); + if (active.size) { + sourceFrame = await inputDecoder.decode(packet, 2); + ingressDecodes++; + } + // Demand phases align with input GOPs; real upstream PLI/network delay is not measured. + } else { + sourceFrame = rawFrame(input, index); + } + const afterInput = now(); + try { + for (const layer of active) { + outputs.set(layer, encoders[layer].encode(sourceFrame, keyframes.has(layer)).then(({ chunk }) => { + encoded[layer]++; + generatedBytes[layer] += chunk.byteLength; + return chunk; + })); + } + demand.forEach((layer, child) => { deliveries[child] ??= deliver(layer, child, outputs.get(layer)); }); + await Promise.all(outputs.values()); + } finally { sourceFrame?.close(); } + const afterEncode = now(); + const decodedAt = await Promise.all(deliveries); + const afterDecode = Math.max(...decodedAt); + const row = { management: afterManagement - tick, input: afterInput - afterManagement, + encodeReady: afterEncode - afterInput, allReceivers: afterDecode - tick, processing: now() - tick }; + for (const key of Object.keys(metrics)) metrics[key].push(row[key]); + if (!phases[phase]) phases[phase] = { demand, frames: 0, work: [] }; + phases[phase].frames++; + phases[phase].work.push(row.processing); + previous = demand; + const delay = start + (index + 1) * 1000 / fps - now(); + if (delay > 0) await new Promise((resolve) => setTimeout(resolve, delay)); + } + const expected = role === "host" ? policy === "exact" ? [96, 32, 160] : [224, 192, 160] + : policy === "exact" ? [96, 32, 0] : [128, 96, 0]; + assert(encoded.every((count, layer) => count === expected[layer]), "Unexpected encode count"); + assert(decoded.every((count) => count === 224), "Missing receiver frames"); + assert(ingressDecodes === (role === "relay" ? 128 : 0), "Repeated or unnecessary input decode"); + return { codec, policy, role, encoded, decoded, ingressDecodes, creates, generatedBytes, selectedBytes, switches, + elapsedMs: now() - start, metrics: Object.fromEntries(Object.entries(metrics).map(([key, values]) => [key, summarize(values)])), + overFrameBudget: metrics.processing.filter((value) => value > 1000 / fps).length, + phases: phases.map(({ demand, frames, work }) => ({ demand, frames, work: summarize(work) })) }; + } finally { + for (const slot of [...encoders, ...receivers, inputDecoder]) if (slot && slot.value.state !== "closed") slot.value.close(); + } +} + +self.onmessage = async () => { + try { + const input = makeInput(); + const results = []; + const support = {}; + for (const codec of ["vp8", "avc1.420028"]) { + support[codec] = (await VideoEncoder.isConfigSupported(config(codec, 2))).supported; + if (!support[codec]) continue; + const ingress = await prepareIngress(codec, input); + for (let round = 1; round <= 2; round++) { + for (const role of ["host", "relay"]) for (const policy of round === 1 ? ["exact", "envelope"] : ["envelope", "exact"]) { + self.postMessage({ progress: `${codec} ${round} ${role} ${policy}` }); + results.push({ round, ...await run(codec, policy, role, input, ingress) }); + } + } + } + self.postMessage({ completed: true, scope: "WebCodecs software preference; independent child decode; no RTP, network, hardware or perceptual acceptance", + userAgent: navigator.userAgent, support, profiles, fps, phaseFrames, trace, results }); + } catch (error) { self.postMessage({ error: String(error.stack ?? error) }); } +}; diff --git a/scripts/package-app-release.mjs b/scripts/package-app-release.mjs index 2e38e10..3506386 100644 --- a/scripts/package-app-release.mjs +++ b/scripts/package-app-release.mjs @@ -8,7 +8,6 @@ import { existsSync, lstatSync, mkdirSync, - mkdtempSync, readFileSync, readdirSync, realpathSync, @@ -16,12 +15,12 @@ import { statSync, writeFileSync, } from "node:fs"; -import { tmpdir } from "node:os"; import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path"; import { fileURLToPath } from "node:url"; import { writeServerLicenseNotices } from "./package-licenses.mjs"; import { tarExecutable } from "./archive-tool.mjs"; +import { resetBuildWorkspace } from "./build-workspace.mjs"; // The Hosted deployment target. deploy/release-app.sh runs the archived binary // as the service user, so the release is always built for linux/amd64, and @@ -192,7 +191,7 @@ assertCleanRevision(repositoryRoot, revision); const mainAsset = mainAssetOf(join(repositoryRoot, CLIENT_DIST)); const releaseId = revision.slice(0, 7); -const temporaryRoot = mkdtempSync(join(tmpdir(), `screener-app-${releaseId}-`)); +const temporaryRoot = resetBuildWorkspace(repositoryRoot, "app-package", "assembly"); const runtimeRoot = join(temporaryRoot, "runtime"); const verifyRoot = join(temporaryRoot, "verify"); diff --git a/scripts/package-client-candidate.mjs b/scripts/package-client-candidate.mjs index 1b8e86f..5f02462 100644 --- a/scripts/package-client-candidate.mjs +++ b/scripts/package-client-candidate.mjs @@ -13,7 +13,7 @@ import { rmSync, writeFileSync, } from "node:fs"; -import { networkInterfaces, tmpdir } from "node:os"; +import { networkInterfaces } from "node:os"; import { createServer } from "node:net"; import { basename, dirname, isAbsolute, join, relative, resolve } from "node:path"; import { Readable } from "node:stream"; @@ -22,6 +22,7 @@ import { fileURLToPath } from "node:url"; import { clientPackageTarget, CLOUDFLARED_VERSION } from "./client-package-targets.mjs"; import { tarExecutable } from "./archive-tool.mjs"; +import { resetBuildWorkspace } from "./build-workspace.mjs"; function fail(message) { throw new Error(message); @@ -254,7 +255,7 @@ async function verifyPackage(root, target, revision, temporaryRoot) { if (target.captureName) { const capture = join(root, "runtime", "native", target.captureName); const probe = JSON.parse(run(capture, ["--probe"], root)); - if (probe?.protocol !== 4) fail("Packaged native capture probe is invalid"); + if (probe?.protocol !== 7) fail("Packaged native capture probe is invalid"); } await verifyLocalPackage(root, target, temporaryRoot); } @@ -322,9 +323,7 @@ if (process.platform !== target.nodePlatform || process.arch !== target.nodeArch assertOutsideRepository(repositoryRoot, outputRoot); const revision = run("git", ["rev-parse", "HEAD"], repositoryRoot).toLowerCase(); -const temporaryRoot = join(tmpdir(), "screener-client-candidate", target.id); -rmSync(temporaryRoot, { recursive: true, force: true }); -mkdirSync(temporaryRoot, { recursive: true, mode: 0o700 }); +const temporaryRoot = resetBuildWorkspace(repositoryRoot, "client-package", target.id, "candidate"); try { const tunnelDownload = join(temporaryRoot, target.tunnelAsset); await download( diff --git a/scripts/package-licenses.mjs b/scripts/package-licenses.mjs index b47ed8c..ea9b47b 100644 --- a/scripts/package-licenses.mjs +++ b/scripts/package-licenses.mjs @@ -4,6 +4,7 @@ import { copyFileSync, existsSync, mkdirSync, readFileSync, readdirSync, writeFi import { createRequire } from "node:module"; import { dirname, join, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; +import { clientGoEnvironment } from "./client-package-targets.mjs"; const root = resolve(dirname(fileURLToPath(import.meta.url)), ".."); const pinned = JSON.parse(readFileSync(join(root, "licenses", "upstream.json"), "utf8")); @@ -13,7 +14,7 @@ const section = (label, body) => `\n===== ${label} =====\n\n${body.trim()}\n`; // The packages the Vite bundle ships to the Browser. Nothing installs // node_modules at runtime any more, so this list is the notice contract rather // than package.json's dependencies field. -const WEB_BUNDLE_PACKAGES = ["react", "react-dom", "livekit-client", "sdp-transform", "zod"]; +const WEB_BUNDLE_PACKAGES = ["react", "react-dom", "sdp-transform", "zod"]; // Both binaries embed the Web bundle, which serves its own notice file. const WEB_NOTICE_LINE = "Web dependencies: served at /third-party-licenses.txt\n"; @@ -72,7 +73,7 @@ export function writeWebLicenseNotices(repositoryRoot, outputFile) { // without a recognised license filename fails closed. function goNotices(repositoryRoot, goCommand, target, command) { const options = { cwd: repositoryRoot, encoding: "utf8", windowsHide: true, - env: { ...process.env, GOOS: target.goos, GOARCH: target.goarch, CGO_ENABLED: "0" } }; + env: clientGoEnvironment(target) }; const runGo = (args) => execFileSync(goCommand, args, options).trim(); const goroot = runGo(["env", "GOROOT"]); const template = '{{if .Module}}{{if not .Module.Main}}[{{printf "%q" .Module.Path}},{{printf "%q" .Module.Version}},{{printf "%q" .Module.Dir}},{{printf "%q" .Dir}}]{{end}}{{end}}'; @@ -139,8 +140,8 @@ export function writeClientLicenseNotices(repositoryRoot, packageRoot, goCommand if (target.goos === "windows" && existsSync(join(packageRoot, "runtime", "native", target.captureName))) { const dependencies = JSON.parse(readFileSync(join(repositoryRoot, "native", "capture", - "windows", "libvpx-dependencies.json"), "utf8")); - const name = `libvpx@${dependencies.libvpx.version}`; + "windows", "webrtc-dependencies.json"), "utf8")); + const name = `webrtc@${dependencies.webrtc.version}`; text += section(name, pinnedNotice(name)); } writeFileSync(join(packageRoot, "THIRD-PARTY-NOTICES.txt"), text + moduleSections(notices.modules)); diff --git a/scripts/required-project-paths.txt b/scripts/required-project-paths.txt index efe2462..a9fa906 100644 --- a/scripts/required-project-paths.txt +++ b/scripts/required-project-paths.txt @@ -67,4 +67,3 @@ scripts/check-project-state.ps1 scripts/check-docs.mjs scripts/install-hooks.sh scripts/install-hooks.ps1 -deploy/coturn/turnserver.conf.example diff --git a/src/client/App.tsx b/src/client/App.tsx index c781686..3b578ba 100644 --- a/src/client/App.tsx +++ b/src/client/App.tsx @@ -26,6 +26,7 @@ import { Btn, Pill } from "./components/living/primitives"; import { Comic, type ComicKind } from "./components/living/Comic"; import { Glyph, type GlyphName } from "./ui/icons"; import { useCopy } from "./ui/copy"; +import { installBrowserDebug } from "./lib/debug"; import { OverlayPreviewPage } from "./pages/OverlayPreviewPage"; import { TooltipPreviewPage } from "./pages/TooltipPreviewPage"; @@ -81,6 +82,7 @@ function readableError(error: unknown, t: (key: "gate.connectFailed") => string) export function App() { const { lang, vis } = useCopy(); + useEffect(installBrowserDebug, []); useEffect(() => { if (appRoute.kind !== "client" && !clientLaunchBootstrap?.launchedByClient) return; const controller = new AbortController(); diff --git a/src/client/components/living/primitives.tsx b/src/client/components/living/primitives.tsx index 3715d42..95f7f79 100644 --- a/src/client/components/living/primitives.tsx +++ b/src/client/components/living/primitives.tsx @@ -55,6 +55,7 @@ export function Btn({ tone, title, disabled, + busy, pressed, expanded, controls, @@ -68,6 +69,7 @@ export function Btn({ tone?: "primary" | "danger" | "on"; title: CopyKey; disabled?: boolean; + busy?: boolean; pressed?: boolean; expanded?: boolean; controls?: string; @@ -90,6 +92,7 @@ export function Btn({ aria-label={label} disabled={disabled && !softDisabled} aria-disabled={softDisabled || undefined} + aria-busy={busy || undefined} aria-pressed={pressed} aria-expanded={expanded} aria-controls={controls} @@ -105,7 +108,12 @@ export function Btn({ : onClick } > - + {cap ? : null} ); diff --git a/src/client/lib/debug.ts b/src/client/lib/debug.ts new file mode 100644 index 0000000..c3600dd --- /dev/null +++ b/src/client/lib/debug.ts @@ -0,0 +1,168 @@ +import { + SIGNALING_PROTOCOL, + type QualityResolution, +} from "../../shared/protocol"; + +interface DebugDetails { + role?: "host" | "viewer"; + type?: string; + state?: string; + generation?: number; + revision?: number; + code?: string | number; + count?: number; + reconnecting?: boolean; + native?: boolean; + resolution?: QualityResolution; + maxFramerate?: number; + maxBitrate?: number; + audio?: boolean; + errorName?: string; +} + +interface BrowserDebugEvent { + at: string; + scope: string; + event: string; + details: DebugDetails; +} + +export const browserDebugEnabled = + typeof window !== "undefined" && + new URLSearchParams(window.location.search).get("debug") === "1"; +const events: BrowserDebugEvent[] = []; +const fields = [ + "role", + "type", + "state", + "generation", + "revision", + "code", + "count", + "reconnecting", + "native", + "resolution", + "maxFramerate", + "maxBitrate", + "audio", + "errorName", +] as const satisfies readonly (keyof DebugDetails)[]; +const errorNames = new Set([ + "Error", + "TypeError", + "RangeError", + "SyntaxError", + "ReferenceError", + "EvalError", + "URIError", + "AbortError", + "NotAllowedError", + "NotFoundError", + "NotReadableError", + "NotSupportedError", + "OverconstrainedError", + "InvalidStateError", + "SecurityError", + "TimeoutError", + "OperationError", + "NetworkError", +]); + +export function debugEvent( + scope: string, + event: string, + details: DebugDetails = {}, +): void { + if ( + !browserDebugEnabled || + !/^[a-z][a-z-]{0,47}$/.test(scope) || + !/^[a-z][a-z-]{0,47}$/.test(event) + ) + return; + const safe: Record = {}; + for (const key of fields) { + const value = details[key]; + if ( + (typeof value === "string" && /^[A-Za-z0-9_-]{1,64}$/.test(value)) || + (typeof value === "number" && Number.isFinite(value)) || + typeof value === "boolean" + ) { + safe[key] = value; + } + } + const record: BrowserDebugEvent = { + at: new Date().toISOString(), + scope, + event, + details: safe, + }; + events.push(record); + if (events.length > 256) events.shift(); + console.info("[screener]", record); +} + +export function debugError( + scope: string, + event: string, + error: unknown, + details: DebugDetails = {}, +): void { + if (!browserDebugEnabled) return; + let errorName = "Error"; + try { + if ( + (error instanceof Error || error instanceof DOMException) && + errorNames.has(error.name) + ) + errorName = error.name; + } catch { + /* Diagnostic inspection must not throw into media work. */ + } + debugEvent(scope, event, { ...details, errorName }); +} + +function snapshot(): BrowserDebugEvent[] { + return structuredClone(events); +} + +export function installBrowserDebug(): (() => void) | undefined { + if (!browserDebugEnabled) return; + window.__SCREENER_DEBUG__ = { + events: snapshot, + clear: () => { + events.length = 0; + }, + export: () => + JSON.stringify( + { + capturedAt: new Date().toISOString(), + protocol: SIGNALING_PROTOCOL, + asset: new URL(import.meta.url).pathname.split("/").at(-1), + events: snapshot(), + }, + null, + 2, + ), + }; + const reportError = (event: ErrorEvent) => + debugError("window", "error", event.error); + const reportRejection = (event: PromiseRejectionEvent) => + debugError("window", "unhandled-rejection", event.reason); + window.addEventListener("error", reportError); + window.addEventListener("unhandledrejection", reportRejection); + debugEvent("app", "loaded"); + return () => { + window.removeEventListener("error", reportError); + window.removeEventListener("unhandledrejection", reportRejection); + }; +} + +declare global { + interface Window { + __SCREENER_DEBUG__?: { + events: () => readonly BrowserDebugEvent[]; + clear: () => void; + export: () => string; + }; + } +} diff --git a/src/client/lib/display-name.ts b/src/client/lib/display-name.ts index 92813b8..3da3002 100644 --- a/src/client/lib/display-name.ts +++ b/src/client/lib/display-name.ts @@ -29,31 +29,18 @@ export function readDisplayName( return readStoredDisplayName() ?? fallback; } -function suffixedDefaultName(prefix: string, clientId: string): DisplayName { - const suffix = clientId.slice(-6); - return ( - normalizeDisplayName(suffix ? `${prefix} (${suffix})` : prefix) ?? - (prefix as DisplayName) - ); -} - export function defaultViewerDisplayName( - clientId: string, visual: boolean, ): DisplayName { return visual - ? suffixedDefaultName("👤", clientId) + ? ("👤" as DisplayName) : (say("common.name.viewerDefault") as DisplayName); } export function defaultHostDisplayName( - clientId: string, visual = false, ): DisplayName { - return suffixedDefaultName( - visual ? "👑" : say("common.name.hostDefault"), - clientId, - ); + return (visual ? "👑" : say("common.name.hostDefault")) as DisplayName; } export function saveDisplayName( diff --git a/src/client/lib/session.ts b/src/client/lib/session.ts index 09edcbd..e980c19 100644 --- a/src/client/lib/session.ts +++ b/src/client/lib/session.ts @@ -36,6 +36,16 @@ export interface HostRoomState extends HostRoomIdentity { inviteUrl: string | null; } +interface HostRoomClaim { + room: HostRoomIdentity; + ready: Promise; + done: Promise; + release: () => void; +} + +let hostRoomClaim: HostRoomClaim | null = null; +let hostRoomRelease: Promise = Promise.resolve(); + export interface ViewerRoute { roomId: string; viewerGrant?: string; @@ -131,18 +141,46 @@ export function takeClientLaunchBootstrap(): ClientLaunchBootstrap { return result; } -export function clearHostRoom(): void { - try { - window.localStorage.removeItem(HOST_ROOM_STORAGE_KEY); - } catch { - // Storage can be disabled; the current page still keeps its in-memory room. +export function clearHostRoom(keepResumeHint = false): void { + const room = readStoredHostRoom("sessionStorage") ?? hostRoomClaim?.room; + if (room) { + removeStoredHostRoom("sessionStorage", room); + if (!keepResumeHint) removeStoredHostRoom("localStorage", room); } + releaseHostRoom(); } -export function readHostRoom(): HostRoomIdentity | null { +export function releaseHostRoom(): void { + if (hostRoomClaim) { + hostRoomClaim.release(); + hostRoomRelease = hostRoomClaim.done; + } + hostRoomClaim = null; +} + +export async function readHostRoom(): Promise { + const tabRoom = readStoredHostRoom("sessionStorage") ?? hostRoomClaim?.room; + const room = + tabRoom ?? (hostRoomLocks() ? readStoredHostRoom("localStorage") : null); + if (!room) return null; + const claim = claimHostRoom(room, Boolean(tabRoom)); + const acquired = await claim.ready; + if (hostRoomClaim !== claim) return null; + if (!acquired) { + releaseHostRoom(); + removeStoredHostRoom("sessionStorage", room); + return null; + } + storeHostRoom("sessionStorage", room); + return room; +} + +function readStoredHostRoom( + storage: "localStorage" | "sessionStorage", +): HostRoomIdentity | null { let stored: string | null; try { - stored = window.localStorage.getItem(HOST_ROOM_STORAGE_KEY); + stored = window[storage].getItem(HOST_ROOM_STORAGE_KEY); } catch { return null; } @@ -153,7 +191,7 @@ export function readHostRoom(): HostRoomIdentity | null { try { const parsed = hostRoomStorageSchema.safeParse(JSON.parse(stored)); if (!parsed.success) { - clearHostRoom(); + window[storage].removeItem(HOST_ROOM_STORAGE_KEY); return null; } const room: HostRoomIdentity = { @@ -165,11 +203,102 @@ export function readHostRoom(): HostRoomIdentity | null { }; return room; } catch { - clearHostRoom(); + try { + window[storage].removeItem(HOST_ROOM_STORAGE_KEY); + } catch { + /* Storage is optional. */ + } return null; } } +function sameHostRoom( + left: HostRoomIdentity | null, + right: HostRoomIdentity, +): boolean { + return left?.roomId === right.roomId && left.hostToken === right.hostToken; +} + +function removeStoredHostRoom( + storage: "localStorage" | "sessionStorage", + room: HostRoomIdentity, +): void { + if (!sameHostRoom(readStoredHostRoom(storage), room)) return; + try { + window[storage].removeItem(HOST_ROOM_STORAGE_KEY); + } catch { + /* Storage is optional. */ + } +} + +function hostRoomLocks(): LockManager | undefined { + try { + return window.navigator?.locks; + } catch { + return undefined; + } +} + +function claimHostRoom( + room: HostRoomIdentity, + allowWithoutLocks: boolean, +): HostRoomClaim { + let claim = hostRoomClaim; + if (!claim || !sameHostRoom(claim.room, room)) { + releaseHostRoom(); + const previousRelease = hostRoomRelease; + let release!: () => void; + let resolveReady!: (ready: boolean) => void; + const lifetime = new Promise((resolve) => { + release = resolve; + }); + claim = { + room, + release, + done: Promise.resolve(), + ready: new Promise((resolve) => { + resolveReady = resolve; + }), + }; + hostRoomClaim = claim; + const locks = hostRoomLocks(); + const requestedClaim = claim; + claim.done = (async () => { + await previousRelease; + if (hostRoomClaim !== requestedClaim) { + resolveReady(false); + return; + } + if (!locks) { + resolveReady(allowWithoutLocks); + return; + } + // The lock follows the room incarnation without exposing the Host token + // in queryable lock names. The browser releases it when this document ends. + const digest = await crypto.subtle.digest( + "SHA-256", + new TextEncoder().encode(room.hostToken), + ); + if (hostRoomClaim !== requestedClaim) { + resolveReady(false); + return; + } + const identity = Array.from(new Uint8Array(digest), (byte) => + byte.toString(16).padStart(2, "0"), + ).join(""); + await locks.request( + `screener:host-room:${room.roomId}:${identity}`, + { ifAvailable: true }, + async (lock) => { + resolveReady(lock !== null); + if (lock) await lifetime; + }, + ); + })().catch(() => resolveReady(allowWithoutLocks)); + } + return claim; +} + export function readPreferredRoomId(): string | null { let stored: string | null; try { @@ -215,22 +344,48 @@ export function clearPreferredRoom(): void { } } -export function writeHostRoom( +export async function writeHostRoom( room: HostRoomIdentity | CreateRoomResponse, -): void { - try { - const canonicalUrl = + owns: () => boolean = () => true, +): Promise { + if (!owns()) return false; + const identity: HostRoomIdentity = { + roomId: room.roomId, + hostToken: room.hostToken, + expiresAt: room.expiresAt, + roomLeaseSeconds: room.roomLeaseSeconds, + canonicalUrl: "canonicalUrl" in room ? room.canonicalUrl - : canonicalViewerUrl(room.inviteUrl); - window.localStorage.setItem( + : canonicalViewerUrl(room.inviteUrl), + }; + const previousClaim = hostRoomClaim; + const claim = claimHostRoom(identity, true); + const acquired = await claim.ready; + if (hostRoomClaim !== claim) return false; + if (!acquired || !owns()) { + if (!acquired || claim !== previousClaim) releaseHostRoom(); + return false; + } + claim.room = identity; + storeHostRoom("sessionStorage", identity); + storeHostRoom("localStorage", identity); + return true; +} + +function storeHostRoom( + storage: "localStorage" | "sessionStorage", + room: HostRoomIdentity, +): void { + try { + window[storage].setItem( HOST_ROOM_STORAGE_KEY, JSON.stringify({ roomId: room.roomId, hostToken: room.hostToken, expiresAt: room.expiresAt, roomLeaseSeconds: room.roomLeaseSeconds, - inviteUrl: canonicalUrl, + inviteUrl: room.canonicalUrl, }), ); } catch { diff --git a/src/client/lib/signaling.ts b/src/client/lib/signaling.ts index 6f0e7ac..55ba2d1 100644 --- a/src/client/lib/signaling.ts +++ b/src/client/lib/signaling.ts @@ -9,6 +9,7 @@ import { } from "../../shared/protocol"; import type { SignalConnectionState } from "../types"; import { qualitySettingsEqual } from "../media/quality"; +import { debugEvent } from "./debug"; type WithoutProtocolEnvelope = T extends { type: string; @@ -102,6 +103,7 @@ export class SignalingClient { return; } this.attachVisibilityListener(); + debugEvent("signal", "start", { role: this.identity.role }); this.events.onStatus(this.reconnectAttempt === 0 ? "connecting" : "reconnecting"); this.connect(); } @@ -119,6 +121,7 @@ export class SignalingClient { socket.close(1000, "client closed"); } this.events.onStatus("offline"); + debugEvent("signal", "stop", { role: this.identity.role }); } send(message: ClientMessage): boolean { @@ -215,6 +218,7 @@ export class SignalingClient { } const generation = ++this.socketGeneration; + debugEvent("signal", "connecting", { role: this.identity.role, generation }); const socket = new WebSocket(signalUrl()); this.socket = socket; @@ -251,6 +255,7 @@ export class SignalingClient { try { message = decodeServerMessage(event.data); } catch { + debugEvent("signal", "invalid-message", { role: this.identity.role }); this.terminateForProtocolMismatch(); return; } @@ -260,6 +265,12 @@ export class SignalingClient { return; } + if (message.type === "error") { + debugEvent("signal", "server-error", { role: this.identity.role, code: message.code }); + } else if (message.type === "route-update" || message.type === "route-status") { + debugEvent("signal", "route", { role: this.identity.role, type: message.type, revision: message.revision }); + } + if ( message.type === "error" && message.code === "INVALID_MESSAGE" && @@ -291,6 +302,7 @@ export class SignalingClient { } } this.events.onStatus("connected"); + debugEvent("signal", "authenticated", { role: this.identity.role, generation }); this.refreshSignalingWatchdog(); } if ( @@ -316,6 +328,8 @@ export class SignalingClient { } this.socket = null; this.authenticated = false; + debugEvent("signal", "closed", { role: this.identity.role, code: event.code, + reconnecting: !this.stopped && shouldReconnectSignaling(event.code) }); this.clearSignalingWatchdog(); this.clearAuthenticationTimer(); if (!this.stopped && shouldReconnectSignaling(event.code)) { diff --git a/src/client/media/host-sfu-route.ts b/src/client/media/host-sfu-route.ts index e064459..134c3f3 100644 --- a/src/client/media/host-sfu-route.ts +++ b/src/client/media/host-sfu-route.ts @@ -4,6 +4,7 @@ import type { MediaRoutePhase, ParticipantRouteAssignment, ServerMessage, + SfuSignalMessage, } from "../../shared/protocol"; import { SfuPublisher, @@ -21,8 +22,10 @@ import { type RouteUpdateResult, } from "./route-transition"; -interface HostPublisherTransport { +export interface HostPublisherTransport { connect(config: SfuConnectionConfig): Promise; + updateConfig(config: SfuConnectionConfig): void; + acceptSignal(message: SfuSignalMessage): Promise; activate( stream: MediaStream, profile: QualityProfile, @@ -39,6 +42,7 @@ interface HostPublisherTransport { interface HostPublisherSlot { revision: number; + connectionId: string; publicationGeneration: string; publisher: HostPublisherTransport; connected: boolean; @@ -86,6 +90,9 @@ export class HostSfuRoute { if (result === "stale") { return result; } + if (this.active) { + this.active.publisher.updateConfig({ ...this.active, revision: update.revision }); + } const pausedActiveReconciliation = this.paused && update.phase === "active" && result !== "duplicate"; const retiredActive = pausedActiveReconciliation @@ -208,12 +215,17 @@ export class HostSfuRoute { !phase || !assignment || !publicationGeneration || + message.publicationGeneration !== publicationGeneration || (phase === "prepare" && candidate?.transport !== "sfu") ) { return; } - if (this.active?.publicationGeneration === publicationGeneration) { + if ( + this.active?.publicationGeneration === publicationGeneration && + this.active.connectionId === message.connectionId + ) { this.active.revision = message.revision; + this.active.publisher.updateConfig(message); if (phase === "active" && this.active.active) { await this.queueActivation(token); } @@ -222,8 +234,10 @@ export class HostSfuRoute { if ( this.pending?.revision === message.revision && this.pending.publicationGeneration === publicationGeneration && + this.pending.connectionId === message.connectionId && !this.pending.failed ) { + this.pending.publisher.updateConfig(message); return; } @@ -235,11 +249,13 @@ export class HostSfuRoute { (metrics) => this.handlePublisherStats(slot, metrics), ) ?? new SfuPublisher({ + send: this.events.send, onDisconnected: () => this.failPublisherSlot(slot), onStats: (metrics) => this.handlePublisherStats(slot, metrics), }); slot = { revision: message.revision, + connectionId: message.connectionId, publicationGeneration, publisher, connected: false, @@ -248,10 +264,7 @@ export class HostSfuRoute { }; this.pending = slot; try { - const connected = await publisher.connect({ - url: message.url, - token: message.token, - }); + const connected = await publisher.connect(message); if (!connected) { if (this.pending === slot && this.route.owns(token)) { this.handleFailure(slot); @@ -307,6 +320,15 @@ export class HostSfuRoute { } } + async acceptSignal(message: SfuSignalMessage): Promise { + const slot = [this.pending, this.active].find((slot) => + slot && !slot.failed && slot.connectionId === message.connectionId && + slot.publicationGeneration === message.publicationGeneration, + ); + if (slot === this.pending && slot?.revision !== message.revision) return; + await slot?.publisher.acceptSignal(message); + } + updateProfile(profile: QualityProfile): Promise { const slots = this.publishingSlots(); return slots.length === 0 @@ -454,9 +476,11 @@ export class HostSfuRoute { if ( this.active && !this.active.failed && - this.active.publicationGeneration === publicationGeneration + this.active.publicationGeneration === publicationGeneration && + this.pending?.publicationGeneration !== publicationGeneration ) { this.active.revision = token.revision; + this.active.publisher.updateConfig(this.active); this.clearPending(); if (this.active.active) { this.commitMedia(token); diff --git a/src/client/media/quality.ts b/src/client/media/quality.ts index 3c586d5..0a06227 100644 --- a/src/client/media/quality.ts +++ b/src/client/media/quality.ts @@ -8,6 +8,7 @@ import { } from "../../shared/protocol"; import { joinItems, say, type CopyKey } from "../ui/copy"; import { displayMediaOptions } from "./audio-capture"; +import { debugError, debugEvent } from "../lib/debug"; export type { DegradationPreference, @@ -189,9 +190,14 @@ export async function captureDisplay( throw new Error(say("host.capture.unavailable")); } - const stream = await navigator.mediaDevices.getDisplayMedia( - displayMediaOptions(captureConstraints(profile)), - ); + debugEvent("capture", "requested", { resolution: profile.resolution, maxFramerate: profile.maxFramerate }); + let stream: MediaStream; + try { + stream = await navigator.mediaDevices.getDisplayMedia(displayMediaOptions(captureConstraints(profile))); + } catch (error) { + debugError("capture", "failed", error); + throw error; + } const videoTrack = stream.getVideoTracks()[0]; if (!videoTrack) { @@ -202,6 +208,7 @@ export async function captureDisplay( for (const audioTrack of stream.getAudioTracks()) { audioTrack.contentHint = "music"; } + debugEvent("capture", "started", { audio: stream.getAudioTracks().length > 0 }); return stream; } @@ -232,7 +239,14 @@ export async function applyVideoCaptureProfile( if (!videoTrackOwnsCaptureConstraints(track)) { return; } - await track.applyConstraints(captureConstraints(profile)); + debugEvent("quality", "capture-requested", { resolution: profile.resolution, maxFramerate: profile.maxFramerate }); + try { + await track.applyConstraints(captureConstraints(profile)); + debugEvent("quality", "capture-applied", { resolution: profile.resolution, maxFramerate: profile.maxFramerate }); + } catch (error) { + debugError("quality", "capture-failed", error); + throw error; + } } function videoTrackOwnsCaptureConstraints(track: MediaStreamTrack): boolean { diff --git a/src/client/media/sender-quality-evidence.ts b/src/client/media/sender-quality-evidence.ts index 029ffd9..688fff3 100644 --- a/src/client/media/sender-quality-evidence.ts +++ b/src/client/media/sender-quality-evidence.ts @@ -46,13 +46,14 @@ function emitUnknownOnce(identities: Set, identity: string): boolean { function nativeQualityState( metrics: ConnectionMetrics, ): "unknown" | "healthy" | "degraded" { + const frames = metrics.intervalFramesEncoded ?? metrics.intervalFramesSent; if ( (metrics.nativeEdgeQualityState !== "healthy" && metrics.nativeEdgeQualityState !== "degraded") || metrics.sampleWindowMs === null || metrics.sampleTimestampMs === null || - metrics.intervalFramesEncoded === null || - metrics.intervalFramesEncoded <= 0 + frames === null || + frames <= 0 ) { return "unknown"; } @@ -92,18 +93,16 @@ export function senderQualityEvidenceFromSnapshot( type: "sender-quality-evidence", childPeerId: snapshot.peerId, connectionId: snapshot.connectionId, - rtpStatsId: - state === "unknown" ? null : snapshot.metrics.rtpStatsId, + rtpStatsId: state === "unknown" ? null : snapshot.metrics.rtpStatsId, trackIdentifier: - state === "unknown" - ? null - : snapshot.metrics.trackIdentifier, + state === "unknown" ? null : snapshot.metrics.trackIdentifier, sampleTimestampMs, routeRevision, state, diagnostics: { natTraversalPath: snapshot.metrics.natTraversalPath, - reason: state === "unknown" ? null : snapshot.metrics.qualityLimitationReason, + reason: + state === "unknown" ? null : snapshot.metrics.qualityLimitationReason, framesPerSecond: snapshot.metrics.framesPerSecond, bitrateKbps: snapshot.metrics.bitrateKbps, captureFramesPerSecond: snapshot.metrics.captureFramesPerSecond, @@ -139,10 +138,7 @@ export function sfuPublisherQualityEvidenceFromMetrics( metrics: ConnectionMetrics, routeRevision: number, publicationGeneration: string, -): Extract< - ClientMessage, - { type: "sfu-publisher-quality-evidence" } -> | null { +): Extract | null { const evidence = nativeQualityState(metrics); const rawSampleTimestampMs = metrics.sampleTimestampMs; const state = ownsCurrentEvidenceGeneration( diff --git a/src/client/media/sfu-standby-prewarmer.ts b/src/client/media/sfu-standby-prewarmer.ts deleted file mode 100644 index cf565a5..0000000 --- a/src/client/media/sfu-standby-prewarmer.ts +++ /dev/null @@ -1,65 +0,0 @@ -interface StandbyRoom { - prepareConnection(url: string): Promise; -} - -interface LiveKitModule { - Room: new () => StandbyRoom; -} - -type LiveKitLoader = () => Promise; - -let sharedLiveKitModule: Promise | null = null; - -function loadLiveKit(): Promise { - sharedLiveKitModule ??= import("livekit-client"); - return sharedLiveKitModule; -} - -export class SfuStandbyPrewarmer { - private desiredUrl: string | null = null; - private disposed = false; - private module: Promise | null = null; - private readonly scheduledUrls = new Set(); - private readonly attemptedUrls = new Set(); - - constructor(private readonly load: LiveKitLoader = loadLiveKit) {} - - setUrl(url: string | null | undefined): void { - this.desiredUrl = url ?? null; - if ( - this.disposed || - !url || - this.scheduledUrls.has(url) || - this.attemptedUrls.has(url) - ) { - return; - } - - this.scheduledUrls.add(url); - void this.prepare(url); - } - - dispose(): void { - this.disposed = true; - this.desiredUrl = null; - } - - private async prepare(url: string): Promise { - try { - this.module ??= this.load(); - const { Room } = await this.module; - if (this.disposed || this.desiredUrl !== url) { - return; - } - this.attemptedUrls.add(url); - await new Room().prepareConnection(url); - } catch { - if (!this.disposed && this.desiredUrl === url) { - this.attemptedUrls.add(url); - } - // Standby warming is optional; the existing tokenized cold path remains. - } finally { - this.scheduledUrls.delete(url); - } - } -} diff --git a/src/client/media/viewer-sfu-route.ts b/src/client/media/viewer-sfu-route.ts index 48d5d55..81139cc 100644 --- a/src/client/media/viewer-sfu-route.ts +++ b/src/client/media/viewer-sfu-route.ts @@ -4,6 +4,7 @@ import type { ParticipantRouteAssignment, PreparedRouteCandidate, ServerMessage, + SfuSignalMessage, } from "../../shared/protocol"; import { SfuSubscriber } from "../sfu/subscriber"; import type { ConnectionMetrics } from "../types"; @@ -22,6 +23,9 @@ import { interface ViewerSubscriberTransport { connect(config: SfuConnectionConfig): Promise; + updateConfig(config: SfuConnectionConfig): void; + acceptSignal(message: SfuSignalMessage): Promise; + reconnect(): boolean; activate(): boolean; deactivate(): boolean; armDecodedFrameProof(requireProgress?: boolean): void; @@ -31,6 +35,7 @@ interface ViewerSubscriberTransport { interface ViewerSubscriberSlot { mediaIdentity: string; + connectionId: string; revision: number; phase: MediaRoutePhase; publicationGeneration: string; @@ -109,7 +114,6 @@ export class ViewerSfuRoute { private transitionTail: Promise = Promise.resolve(); private resyncGeneration = 0; private resyncing = false; - private manualReconnectRevision: number | null = null; private subscriberGeneration = 0; private paused = false; private closed = false; @@ -129,18 +133,15 @@ export class ViewerSfuRoute { previousMediaAssignment, update.assignment, ); - const continuingRecovery = - previousRevision !== update.revision && sameCommittedMedia - ? this.manualReconnectRevision !== null - ? "manual" - : this.recovery !== null - ? "automatic" - : null - : null; + const continuingRecovery = previousRevision !== update.revision && + sameCommittedMedia && this.recovery !== null; const result = this.route.accept(update); if (result === "stale") { return result; } + if (this.active) { + this.active.subscriber.updateConfig({ ...this.active, revision: update.revision }); + } if (continuingRecovery) { if (this.active) { this.active.revision = update.revision; @@ -148,9 +149,6 @@ export class ViewerSfuRoute { if (this.pending?.phase === "active") { this.pending.revision = update.revision; } - if (this.manualReconnectRevision !== null) { - this.manualReconnectRevision = update.revision; - } if (this.recovery) { this.recovery.revision = update.revision; if (this.pending === null) { @@ -180,7 +178,6 @@ export class ViewerSfuRoute { } if (previousRevision !== update.revision && !sameCommittedMedia) { this.recovery = null; - this.manualReconnectRevision = null; if ( update.phase !== "prepare" && this.pendingPeerRevision !== null @@ -242,25 +239,7 @@ export class ViewerSfuRoute { if (this.resyncing) { return result; } - const transition = this.queueActiveRoute(token, acknowledge, this.paused); - if ( - this.manualReconnectRevision === update.revision && - this.pending === null - ) { - void transition.then(() => { - if ( - this.closed || - this.paused || - this.route.getRevision() !== update.revision || - this.route.getPhase() !== "active" - ) { - return; - } - if (this.manualReconnectRevision === update.revision) { - this.events.send({ type: "refresh-sfu", revision: update.revision }); - } - }); - } + void this.queueActiveRoute(token, acknowledge, this.paused); return result; } @@ -297,7 +276,6 @@ export class ViewerSfuRoute { if ( this.closed || this.resyncing || - this.manualReconnectRevision !== null || this.pending !== null || this.paused || revision === null || @@ -312,17 +290,10 @@ export class ViewerSfuRoute { return false; } - this.manualReconnectRevision = revision; - if (!this.events.send({ type: "refresh-sfu", revision })) { - this.manualReconnectRevision = null; - return false; - } - this.events.onSfuState?.("reconnecting", revision); - return true; + return active.subscriber.reconnect(); } private discardPending(): void { - this.manualReconnectRevision = null; this.pendingPeerRevision = null; this.events.preparePeer?.(null); this.events.prepareChild?.(null); @@ -365,7 +336,6 @@ export class ViewerSfuRoute { const resyncGeneration = ++this.resyncGeneration; this.resyncing = true; - this.manualReconnectRevision = null; this.route.reset(); this.recovery = null; this.events.prepareChild?.(null); @@ -424,13 +394,12 @@ export class ViewerSfuRoute { const phase = this.route.getPhase(); const candidate = this.route.getPreparedCandidate(); const publicationGeneration = assignment?.sfuPublicationGeneration; - const manualReconnect = - phase === "active" && this.manualReconnectRevision === message.revision; if ( !token || !phase || assignment?.upstream.kind !== "sfu" || !publicationGeneration || + message.publicationGeneration !== publicationGeneration || (phase === "prepare" && (candidate?.childPeerId !== this.viewerPeerId || candidate.transport !== "sfu")) @@ -442,8 +411,9 @@ export class ViewerSfuRoute { phase === "active" && this.active?.publicationGeneration === publicationGeneration && this.active.activated && - !manualReconnect + this.active.connectionId === message.connectionId ) { + this.active.subscriber.updateConfig(message); if (phase === "active") { await this.queueActiveRoute(token, true, this.paused); } @@ -452,14 +422,17 @@ export class ViewerSfuRoute { if ( this.pending?.revision === message.revision && this.pending.publicationGeneration === publicationGeneration && + this.pending.connectionId === message.connectionId && !this.pending.failed ) { + this.pending.subscriber.updateConfig(message); return; } this.clearPending(); let slot: ViewerSubscriberSlot; const subscriberEvents = { + send: this.events.send, onStream: (stream: MediaStream | null) => { if (stream) { this.handleStream(slot, stream); @@ -518,6 +491,7 @@ export class ViewerSfuRoute { const preparedCandidate = this.route.getPreparedCandidate(); slot = { mediaIdentity: `${publicationGeneration}:${++this.subscriberGeneration}`, + connectionId: message.connectionId, revision: message.revision, phase, publicationGeneration, @@ -538,10 +512,7 @@ export class ViewerSfuRoute { }; this.pending = slot; try { - const connected = await subscriber.connect({ - url: message.url, - token: message.token, - }); + const connected = await subscriber.connect(message); if (!connected) { const currentToken = this.currentPendingToken(slot, token); if (this.pending === slot && currentToken) { @@ -594,12 +565,20 @@ export class ViewerSfuRoute { ); } + async acceptSignal(message: SfuSignalMessage): Promise { + const slot = [this.pending, this.active].find((slot) => + slot && !slot.failed && slot.connectionId === message.connectionId && + slot.publicationGeneration === message.publicationGeneration, + ); + if (slot === this.pending && slot?.revision !== message.revision) return; + await slot?.subscriber.acceptSignal(message); + } + async disconnect(): Promise { if (this.closed) { return; } this.closed = true; - this.manualReconnectRevision = null; this.resyncGeneration += 1; this.resyncing = false; this.route.reset(); @@ -678,10 +657,11 @@ export class ViewerSfuRoute { this.active?.publicationGeneration === assignment.sfuPublicationGeneration && this.active?.activated && - this.pending?.revision !== token.revision + this.pending?.publicationGeneration !== assignment.sfuPublicationGeneration ) { this.clearPending(); this.active.revision = token.revision; + this.active.subscriber.updateConfig(this.active); this.commitMedia(token); return; } @@ -736,12 +716,10 @@ export class ViewerSfuRoute { ): Promise { const assignment = this.route.getActiveAssignment(); const publicationGeneration = assignment?.sfuPublicationGeneration; - const manualReconnect = this.manualReconnectRevision === token.revision; if ( this.active?.revision === token.revision && this.active.publicationGeneration === publicationGeneration && - this.active.activated && - !manualReconnect + this.active.activated && this.pending === null ) { return; } @@ -843,7 +821,6 @@ export class ViewerSfuRoute { slot.subscriber.stopDecodedFrameProof(); this.pending = null; const previous = this.active; - const manualReconnect = this.manualReconnectRevision === slot.revision; this.active = slot; if (!this.route.markMediaActive(token)) { this.active = previous; @@ -852,9 +829,6 @@ export class ViewerSfuRoute { return; } this.recovery = null; - if (manualReconnect) { - this.manualReconnectRevision = null; - } this.events.onSfuStream(slot.stream, assignment, true, slot.revision); if (previous && previous !== slot) { previous.failed = true; @@ -865,9 +839,6 @@ export class ViewerSfuRoute { } await disconnectSubscriber(previous.subscriber); } - if (manualReconnect) { - this.events.onSfuState?.("connected", slot.revision); - } } private handleVideoAvailability( @@ -957,12 +928,6 @@ export class ViewerSfuRoute { slot.failed = true; return; } - const failedManualReconnect = - wasPending && - this.manualReconnectRevision === slot.revision && - this.active?.publicationGeneration === slot.publicationGeneration && - this.active.activated && - !this.active.failed; slot.failed = true; slot.subscriber.stopDecodedFrameProof(); if (this.pending === slot) { @@ -972,11 +937,6 @@ export class ViewerSfuRoute { this.active = null; this.events.onSfuUpdate?.(null, slot.revision); } - if (failedManualReconnect) { - this.manualReconnectRevision = null; - this.events.onSfuState?.("connected", slot.revision); - return; - } const assignment = this.route.getPlannedAssignment(); const revision = this.route.getRevision(); diff --git a/src/client/native/client.ts b/src/client/native/client.ts index 36f3a90..16ddd54 100644 --- a/src/client/native/client.ts +++ b/src/client/native/client.ts @@ -1,9 +1,11 @@ import type { z } from "zod"; +import { debugEvent } from "../lib/debug"; import type { IceConfig, QualitySettings, SignalPayload, + SfuSignalMessage, } from "../../shared/protocol"; import { createOpaqueId } from "../lib/opaque-id"; import { @@ -21,8 +23,10 @@ import { NATIVE_CLIENT_PROTOCOL, NATIVE_CLIENT_SUBPROTOCOL, pongResponseSchema, + publicationResponseSchema, receiveAnswerResponseSchema, readyResponseSchema, + requestFailedResponseSchema, shareStartedResponseSchema, shareSourceReplacedResponseSchema, shareUpdatedResponseSchema, @@ -115,7 +119,9 @@ export class NativeClient { readonly health: NativeHealth, private readonly socket: WebSocket, ) { - socket.addEventListener("message", (event) => this.handleMessage(event.data)); + socket.addEventListener("message", (event) => + this.handleMessage(event.data), + ); socket.addEventListener("close", () => this.handleClose()); socket.addEventListener("error", () => this.handleClose()); } @@ -123,20 +129,33 @@ export class NativeClient { static async connect(): Promise { const health = await discoverNativeHealth(); if (!health) return null; - const socket = new WebSocket( - `ws://127.0.0.1:${health.port}/control`, - [`${NATIVE_CLIENT_SUBPROTOCOL}.${health.instanceToken}`], - ); + const socket = new WebSocket(`ws://127.0.0.1:${health.port}/control`, [ + `${NATIVE_CLIENT_SUBPROTOCOL}.${health.instanceToken}`, + ]); const opened = await new Promise((resolveOpen) => { - const timer = window.setTimeout(() => resolveOpen(false), REQUEST_TIMEOUT_MS); - socket.addEventListener("open", () => { - window.clearTimeout(timer); - resolveOpen(socket.protocol === `${NATIVE_CLIENT_SUBPROTOCOL}.${health.instanceToken}`); - }, { once: true }); - socket.addEventListener("error", () => { - window.clearTimeout(timer); - resolveOpen(false); - }, { once: true }); + const timer = window.setTimeout( + () => resolveOpen(false), + REQUEST_TIMEOUT_MS, + ); + socket.addEventListener( + "open", + () => { + window.clearTimeout(timer); + resolveOpen( + socket.protocol === + `${NATIVE_CLIENT_SUBPROTOCOL}.${health.instanceToken}`, + ); + }, + { once: true }, + ); + socket.addEventListener( + "error", + () => { + window.clearTimeout(timer); + resolveOpen(false); + }, + { once: true }, + ); }); if (!opened) { socket.close(); @@ -145,6 +164,7 @@ export class NativeClient { const client = new NativeClient(health, socket); try { await client.request("hello", {}, readyResponseSchema); + debugEvent("native", "connected"); return client; } catch { client.close(); @@ -197,7 +217,9 @@ export class NativeClient { if (response.sourceKey !== nativeCaptureTargetKey(target)) { throw new Error("Native capture preview identity changed"); } - return response.data ? `data:${response.mime};base64,${response.data}` : null; + return response.data + ? `data:${response.mime};base64,${response.data}` + : null; } async startShare( @@ -215,10 +237,7 @@ export class NativeClient { return { audio: response.audio, codec: response.codec }; } - async updateShare( - shareId: string, - profile: QualitySettings, - ): Promise { + async updateShare(shareId: string, profile: QualitySettings): Promise { const response = await this.request( "update-share", { shareId, profile }, @@ -280,6 +299,109 @@ export class NativeClient { return { type: "offer", sdp: response.sdp }; } + async preparePublication( + shareId: string, + publicationGeneration: string, + connectionId: string, + iceConfig: IceConfig, + ) { + const response = await this.request( + "prepare-publication", + { + shareId, + publicationGeneration, + connectionId, + iceServers: iceConfig.iceServers.map((server) => ({ + urls: Array.isArray(server.urls) ? server.urls : [server.urls], + })), + }, + publicationResponseSchema, + ); + if ( + response.type !== "publication-offer" || + !response.sdp || + response.shareId !== shareId || + response.publicationGeneration !== publicationGeneration || + response.connectionId !== connectionId + ) { + throw new Error("Native publication identity changed"); + } + return { + description: { type: "offer" as const, sdp: response.sdp }, + media: response.media, + }; + } + + async publicationMedia( + shareId: string, + publicationGeneration: string, + connectionId: string, + ) { + const response = await this.request( + "publication-media", + { + shareId, + publicationGeneration, + connectionId, + }, + publicationResponseSchema, + ); + if ( + response.type !== "publication-media" || + response.shareId !== shareId || + response.publicationGeneration !== publicationGeneration || + response.connectionId !== connectionId + ) { + throw new Error("Native publication identity changed"); + } + return response.media; + } + + async acceptPublicationSignal( + shareId: string, + message: SfuSignalMessage, + ): Promise { + const identity = { + shareId, + publicationGeneration: message.publicationGeneration, + connectionId: message.connectionId, + }; + if ( + message.kind === "description" && + message.description?.type === "answer" + ) { + await this.request( + "publication-answer", + { ...identity, sdp: message.description.sdp }, + nativeAckResponseSchema, + ); + } else if (message.kind === "candidate") { + await this.request( + "publication-candidate", + { ...identity, candidate: message.candidate ?? null }, + nativeAckResponseSchema, + ); + } else if (message.kind === "layers") { + await this.request( + "publication-layers", + { ...identity, activeCount: message.activeCount }, + nativeAckResponseSchema, + ); + } + } + + async closePublication( + shareId: string, + publicationGeneration: string, + connectionId: string, + ): Promise { + await this.request( + "close-publication", + { shareId, publicationGeneration, connectionId }, + nativeAckResponseSchema, + ); + } + async prepareLocalEdge( shareId: string, connectionId: string, @@ -364,11 +486,7 @@ export class NativeClient { } async stopReceive(shareId: string): Promise { - await this.request( - "stop-receive", - { shareId }, - nativeAckResponseSchema, - ); + await this.request("stop-receive", { shareId }, nativeAckResponseSchema); } async acceptSignal( @@ -402,11 +520,7 @@ export class NativeClient { } async stopShare(shareId: string): Promise { - await this.request( - "stop-share", - { shareId }, - nativeAckResponseSchema, - ); + await this.request("stop-share", { shareId }, nativeAckResponseSchema); } async setPaused(shareId: string, paused: boolean): Promise { @@ -439,28 +553,40 @@ export class NativeClient { } const id = createOpaqueId(); return new Promise((resolveRequest, rejectRequest) => { - const timer = timeoutMs === null - ? null - : window.setTimeout(() => { - this.pending.delete(id); - rejectRequest(new Error("Screener Client request timed out")); - }, timeoutMs); + const timer = + timeoutMs === null + ? null + : window.setTimeout(() => { + this.pending.delete(id); + debugEvent("native", "request-timeout", { type }); + rejectRequest(new Error("Screener Client request timed out")); + }, timeoutMs); this.pending.set(id, { schema, - resolve: resolveRequest as (value: unknown) => void, - reject: rejectRequest, + resolve: (value) => { + debugEvent("native", "response", { type }); + resolveRequest(value as T); + }, + reject: (error) => { + debugEvent("native", "request-failed", { type }); + rejectRequest(error); + }, timer, }); try { - this.socket.send(JSON.stringify({ - version: NATIVE_CLIENT_PROTOCOL, - id, - type, - ...fields, - })); + debugEvent("native", "request", { type }); + this.socket.send( + JSON.stringify({ + version: NATIVE_CLIENT_PROTOCOL, + id, + type, + ...fields, + }), + ); } catch { if (timer !== null) window.clearTimeout(timer); this.pending.delete(id); + debugEvent("native", "request-failed", { type }); rejectRequest(new Error("Screener Client request failed")); } }); @@ -480,6 +606,10 @@ export class NativeClient { if (!pending) return; this.pending.delete(id); if (pending.timer !== null) window.clearTimeout(pending.timer); + if (requestFailedResponseSchema.safeParse(value).success) { + pending.reject(new Error("Screener Client request failed")); + return; + } const parsed = pending.schema.safeParse(value); if (parsed.success) { pending.resolve(parsed.data); @@ -493,6 +623,9 @@ export class NativeClient { this.failConnection(); return; } + if (event.data.type === "capture-state" || event.data.type === "edge-state" || event.data.type === "publication-state") { + debugEvent("native", "state", { type: event.data.type, state: event.data.state }); + } else if (event.data.type === "share-ended") debugEvent("native", "share-ended"); for (const listener of this.listeners) { listener(event.data); } @@ -500,6 +633,7 @@ export class NativeClient { private handleClose(): void { if (this.closed) return; + debugEvent("native", "closed"); this.closed = true; this.rejectPending(); this.listeners.clear(); diff --git a/src/client/native/media-bridge.ts b/src/client/native/media-bridge.ts index a3bdc6d..7fb9d52 100644 --- a/src/client/native/media-bridge.ts +++ b/src/client/native/media-bridge.ts @@ -1,5 +1,6 @@ import type { SignalPayload } from "../../shared/protocol"; import { createOpaqueId } from "../lib/opaque-id"; +import { browserDebugEnabled, debugError, debugEvent } from "../lib/debug"; import type { ConnectionMetrics } from "../types"; import { collectConnectionMetrics, @@ -11,6 +12,8 @@ import type { NativeClientEvent } from "./wire"; const BRIDGE_TIMEOUT_MS = 8_000; const MAX_PENDING_CANDIDATES = 64; +export class NativeMediaBridgeError extends Error {} + export interface NativeMediaBridgeControl { prepareLocalEdge( shareId: string, @@ -61,11 +64,11 @@ export class NativeMediaBridge { async start(): Promise { if (this.disposed || this.unsubscribe) { - throw new Error("Native media bridge is unavailable"); + throw new NativeMediaBridgeError("Native media bridge is unavailable"); } const started = new Promise((resolve, reject) => { this.rejectStart = reject; - this.startTimer = window.setTimeout(() => this.fail(), BRIDGE_TIMEOUT_MS); + this.startTimer = window.setTimeout(() => this.fail("timeout"), BRIDGE_TIMEOUT_MS); const complete = () => { if ( !this.ready && @@ -86,16 +89,23 @@ export class NativeMediaBridge { complete(); }); this.peer.addEventListener("connectionstatechange", () => { + debugEvent("native-bridge", "connection-state", { state: this.peer.connectionState }); if ( this.peer.connectionState === "failed" || this.peer.connectionState === "closed" ) { - this.fail(); + this.fail("connection-state"); return; } complete(); }); }); + this.peer.addEventListener("iceconnectionstatechange", () => { + debugEvent("native-bridge", "ice-state", { state: this.peer.iceConnectionState }); + }); + this.peer.addEventListener("icegatheringstatechange", () => { + debugEvent("native-bridge", "ice-gathering-state", { state: this.peer.iceGatheringState }); + }); this.peer.addEventListener("icecandidate", (event) => { void this.control.acceptSignal( this.shareId, @@ -112,12 +122,12 @@ export class NativeMediaBridge { } : null, }, - ).catch(() => this.fail()); + ).catch(() => this.fail("candidate-signal")); }); this.unsubscribe = this.control.onEvent((event) => { if (event.shareId !== this.shareId) return; if (event.type === "share-ended") { - this.fail(); + this.fail("share-ended"); return; } if ( @@ -129,7 +139,7 @@ export class NativeMediaBridge { if (event.type === "edge-candidate") { if (!this.remoteDescriptionSet) { if (this.pendingCandidates.length >= MAX_PENDING_CANDIDATES) { - this.fail(); + this.fail("candidate-overflow"); } else { this.pendingCandidates.push(event.candidate); } @@ -137,12 +147,12 @@ export class NativeMediaBridge { } void this.peer .addIceCandidate(event.candidate) - .catch(() => this.fail()); + .catch(() => this.fail("candidate-apply")); } else if ( event.type === "edge-state" && (event.state === "failed" || event.state === "closed") ) { - this.fail(); + this.fail("native-state"); } }); @@ -179,15 +189,17 @@ export class NativeMediaBridge { try { return await Promise.race([negotiate(), started]); } catch (error) { - this.dispose(); - throw error; + this.fail("negotiation"); + throw error instanceof NativeMediaBridgeError + ? error + : new NativeMediaBridgeError("Native media bridge failed", { cause: error }); } } dispose(): void { if (this.disposed) return; this.disposed = true; - this.rejectStart?.(new Error("Native media bridge failed")); + this.rejectStart?.(new NativeMediaBridgeError("Native media bridge failed")); this.rejectStart = null; this.unsubscribe?.(); this.unsubscribe = null; @@ -200,12 +212,48 @@ export class NativeMediaBridge { ); } - private fail(): void { + private fail(reason: string): void { if (this.disposed) return; + debugEvent("native-bridge", "failed", { type: reason, state: this.ready ? "active" : "starting" }); + void this.reportFailureDiagnostics(); if (this.ready) this.onFailed(); this.dispose(); } + private async reportFailureDiagnostics(): Promise { + if (!browserDebugEnabled) return; + try { + debugEvent("native-bridge", "connection-state", { state: this.peer.connectionState }); + debugEvent("native-bridge", "ice-state", { state: this.peer.iceConnectionState }); + debugEvent("native-bridge", "ice-gathering-state", { state: this.peer.iceGatheringState }); + const receivers = this.peer.getReceivers(); + for (const kind of ["video", "audio"]) { + debugEvent("native-bridge", "receiver-count", { + type: kind, state: "live", + count: receivers.filter((receiver) => receiver.track.kind === kind && receiver.track.readyState === "live").length, + }); + } + for (const transport of new Set(receivers.map((receiver) => receiver.transport))) { + if (transport) debugEvent("native-bridge", "dtls-state", { state: transport.state }); + } + // Start the one snapshot before disposal; it never delays the bridge deadline. + const stats = Array.from((await this.peer.getStats()).values()); + for (const direction of ["local", "remote"]) { + for (const type of ["host", "srflx", "prflx", "relay"]) { + for (const protocol of ["udp", "tcp"]) { + debugEvent("native-bridge", "candidate-count", { + type: `${direction}-${type}`, state: protocol, + count: stats.filter((entry) => entry.type === `${direction}-candidate` && + entry.candidateType === type && entry.protocol === protocol).length, + }); + } + } + } + } catch (error) { + debugError("native-bridge", "diagnostics-failed", error); + } + } + private clearStartTimer(): void { if (this.startTimer === null) return; window.clearTimeout(this.startTimer); diff --git a/src/client/native/media-ingress.ts b/src/client/native/media-ingress.ts index a5bc673..7d8ed97 100644 --- a/src/client/native/media-ingress.ts +++ b/src/client/native/media-ingress.ts @@ -35,6 +35,7 @@ export class NativeMediaIngress { readonly connectionId = createOpaqueId(); readonly source: NativeSenderSource = { connectionId: this.connectionId, + getProfile: () => this.getProfile(), format: () => { const metrics = this.sender?.getSnapshot().metrics; return { width: metrics?.frameWidth ?? null, height: metrics?.frameHeight ?? null }; @@ -55,6 +56,7 @@ export class NativeMediaIngress { readonly shareId: string, private readonly client: NativeMediaIngressControl, private readonly onFailed: () => void, + private readonly getProfile: () => QualityProfile, ) {} get hasAudio(): boolean { return this.audio; } diff --git a/src/client/native/native-sender-edge.ts b/src/client/native/native-sender-edge.ts index 8a1a363..37c400a 100644 --- a/src/client/native/native-sender-edge.ts +++ b/src/client/native/native-sender-edge.ts @@ -1,4 +1,4 @@ -import type { IceConfig, SignalPayload } from "../../shared/protocol"; +import type { IceConfig, QualitySettings, SignalPayload } from "../../shared/protocol"; import { iceServersWithNatPrediction, isNativeNatSurveyCandidate, @@ -11,6 +11,7 @@ import type { NativeClientEvent } from "./wire"; const MAX_PENDING_CANDIDATES = 64; export interface NativeEdgeControl { + updateShare(shareId: string, profile: QualitySettings): Promise; prepareEdge( shareId: string, connectionId: string, diff --git a/src/client/native/native-sender-peer.ts b/src/client/native/native-sender-peer.ts index c0d4244..b58f1ec 100644 --- a/src/client/native/native-sender-peer.ts +++ b/src/client/native/native-sender-peer.ts @@ -1,6 +1,7 @@ import type { IceConfig, PreparedRouteCandidate, + QualitySettings, SignalPayload, } from "../../shared/protocol"; import { EMPTY_METRICS, type PeerSnapshot } from "../types"; @@ -20,6 +21,7 @@ export interface NativeSourceFormat { export interface NativeSenderSource { connectionId: string; + getProfile: () => QualitySettings; format?: () => NativeSourceFormat; } @@ -35,10 +37,10 @@ export class NativeSenderPeer implements HostMediaPeer { constructor( readonly peerId: string, connectionId: string, - shareId: string, + private readonly shareId: string, iceConfig: IceConfig, natPredictionEnabled: boolean, - control: NativeEdgeControl, + private readonly control: NativeEdgeControl, events: NativeSenderPeerEvents, codec: NativeVideoCodec, private readonly source?: NativeSenderSource, @@ -145,7 +147,10 @@ export class NativeSenderPeer implements HostMediaPeer { private readonly events: NativeSenderPeerEvents; - start(): Promise { + async start(): Promise { + if (this.disposed) return false; + if (this.source && !(await this.updateProfile(this.source.getProfile()))) return false; + if (this.disposed) return false; return this.edge.start(); } @@ -189,14 +194,20 @@ export class NativeSenderPeer implements HostMediaPeer { // mutate a live edge's server list behind the route owner's fence. } - updateProfile(_profile: Parameters[0]): Promise { - // The share-level native owner updates the one encoded source before its - // Pion edges. An individual edge cannot own or repeat that operation. - return Promise.resolve(true); + async updateProfile(profile: Parameters[0]): Promise { + if (this.disposed) return false; + // Native Host capture remains share-owned; only received sources need a relay ceiling. + if (!this.source) return true; + try { + await this.control.updateShare(this.shareId, profile); + return !this.disposed; + } catch { + return false; + } } - updateCaptureProfile(_profile: Parameters[0]): Promise { - return Promise.resolve(true); + updateCaptureProfile(profile: Parameters[0]): Promise { + return this.updateProfile(profile); } setPaused(_paused: boolean): void { diff --git a/src/client/native/native-sfu-publisher.ts b/src/client/native/native-sfu-publisher.ts new file mode 100644 index 0000000..dab5576 --- /dev/null +++ b/src/client/native/native-sfu-publisher.ts @@ -0,0 +1,220 @@ +import type { IceConfig, SfuSignalMessage } from "../../shared/protocol"; +import type { HostPublisherTransport } from "../media/host-sfu-route"; +import type { + SfuConnectionConfig, + SfuPublisherFailureStage, +} from "../sfu/publisher"; +import type { NativeClient } from "./client"; +import type { NativeClientEvent } from "./wire"; +import { EMPTY_METRICS, type ConnectionMetrics } from "../types"; + +type PublicationControl = Pick< + NativeClient, + | "preparePublication" + | "publicationMedia" + | "acceptPublicationSignal" + | "closePublication" + | "onEvent" +>; + +// The page owns room signaling. The Client publishes its existing encoded source. +export class NativeSfuPublisher implements HostPublisherTransport { + private config: SfuConnectionConfig | null = null; + private pending: Array< + Pick + > = []; + private unsubscribe: (() => void) | null = null; + private prepared = false; + private closed = false; + private failure: SfuPublisherFailureStage | null = null; + + constructor( + private readonly client: PublicationControl, + private readonly shareId: string, + private readonly iceConfig: IceConfig, + private readonly events: { + send: (message: SfuSignalMessage) => boolean; + onDisconnected: () => void; + onStats?: (metrics: ConnectionMetrics | null) => void; + }, + ) {} + + async connect(config: SfuConnectionConfig): Promise { + if (this.config || this.closed) return false; + this.config = config; + this.unsubscribe = this.client.onEvent((event) => this.onEvent(event)); + return true; + } + + updateConfig(config: SfuConnectionConfig): void { + if ( + this.config?.connectionId !== config.connectionId || + this.config.publicationGeneration !== config.publicationGeneration + ) + return; + this.config = config; + this.flush(); + } + + async activate(): Promise { + const config = this.config; + if (!config || this.prepared || this.closed) return false; + try { + const result = await this.client.preparePublication( + this.shareId, + config.publicationGeneration, + config.connectionId, + this.iceConfig, + ); + if (this.closed) { + await this.client.closePublication( + this.shareId, + config.publicationGeneration, + config.connectionId, + ); + return false; + } + this.pending.unshift({ kind: "description", ...result }); + this.prepared = true; + this.flush(); + return !this.closed; + } catch { + this.fail("video-publish"); + return false; + } + } + + async acceptSignal(message: SfuSignalMessage): Promise { + if ( + this.closed || + message.connectionId !== this.config?.connectionId || + message.publicationGeneration !== this.config.publicationGeneration + ) + return; + try { + await this.client.acceptPublicationSignal(this.shareId, message); + } catch { + this.fail("transport"); + } + } + + async updateProfile(): Promise { + const config = this.config; + if (!config || !this.prepared || this.closed) return false; + try { + const media = await this.client.publicationMedia( + this.shareId, + config.publicationGeneration, + config.connectionId, + ); + if (this.closed) return false; + this.pending = this.pending.filter(({ kind }) => kind !== "media"); + this.pending.push({ kind: "media", media }); + this.flush(); + return !this.closed; + } catch { + this.fail("source"); + return false; + } + } + + replaceStream(): Promise { + return this.updateProfile(); + } + setPaused(): void {} // The shared Native capture owner applies pause to every path. + getFailureStage(): SfuPublisherFailureStage | null { + return this.failure; + } + async deactivate(): Promise { + await this.disconnect(); + return true; + } + + async disconnect(): Promise { + if (this.closed) return; + this.closed = true; + this.unsubscribe?.(); + this.unsubscribe = null; + this.pending = []; + const config = this.config; + if (config) + await this.client.closePublication( + this.shareId, + config.publicationGeneration, + config.connectionId, + ); + } + + private onEvent(event: NativeClientEvent): void { + if ( + this.closed || + event.shareId !== this.shareId || + !("publicationGeneration" in event) || + event.publicationGeneration !== this.config?.publicationGeneration || + event.connectionId !== this.config.connectionId + ) + return; + if (event.type === "publication-quality") { + this.events.onStats?.({ + ...EMPTY_METRICS, + sampleTimestampMs: event.sampleTimestampMs, + sampleWindowMs: event.sampleWindowMs, + rtpStatsId: event.rtpStatsId, + trackIdentifier: event.trackIdentifier, + rtpRid: event.rid || null, + videoEncodingCount: event.videoEncodingCount, + activeVideoEncodingCount: event.activeVideoEncodingCount, + intervalFramesSent: event.intervalFramesSent, + framesPerSecond: event.framesPerSecond, + frameWidth: event.width || null, + frameHeight: event.height || null, + resolution: + event.width && event.height ? `${event.width}x${event.height}` : null, + bitrateKbps: event.bitrateKbps, + audioBitrateKbps: event.audioBitrateKbps, + availableOutgoingKbps: event.availableOutgoingKbps, + codec: `video/${event.codec.toUpperCase()}`, + nativeEdgeQualityState: event.state, + qualityLimitationReason: event.reason, + }); + } else if (event.type === "publication-candidate" && event.candidate) { + if (this.pending.length >= 64) { + this.fail("transport"); + return; + } + this.pending.push({ kind: "candidate", candidate: event.candidate }); + this.flush(); + } else if ( + event.type === "publication-state" && + (event.state === "disconnected" || + event.state === "failed" || + event.state === "closed") + ) { + this.fail("transport"); + } + } + + private flush(): void { + if (this.closed || !this.prepared || !this.config) return; + while (this.pending[0]) { + if ( + !this.events.send({ + type: "sfu-signal", + revision: this.config.revision, + publicationGeneration: this.config.publicationGeneration, + connectionId: this.config.connectionId, + ...this.pending[0], + }) + ) + return; + this.pending.shift(); + } + } + + private fail(stage: SfuPublisherFailureStage): void { + if (this.closed || this.failure) return; + this.failure = stage; + void this.disconnect().catch(() => undefined); + this.events.onDisconnected(); + } +} diff --git a/src/client/native/native-viewer-peer.ts b/src/client/native/native-viewer-peer.ts index 3de7c29..4497a77 100644 --- a/src/client/native/native-viewer-peer.ts +++ b/src/client/native/native-viewer-peer.ts @@ -50,13 +50,13 @@ export class NativeCapableViewerPeer implements ViewerMediaPeer { private identity: { parentPeerId: string; connectionId: string } | null = null; private signalTail: Promise = Promise.resolve(); private currentIceConfig: PeerIceConfig; - private forceBrowserNext = false; constructor( iceConfig: PeerIceConfig, private readonly events: ViewerPeerEvents, private readonly options: ViewerPeerOptions, - private readonly nativeClient: Promise, + private readonly acquireNativeClient: () => Promise, + private readonly onNativeUnavailable: () => void, private readonly nativeSessionId: string, private readonly edgeCapacity: number, ) { @@ -90,10 +90,8 @@ export class NativeCapableViewerPeer implements ViewerMediaPeer { this.identity = nextIdentity; } if (!this.backend) { - const forceBrowser = this.forceBrowserNext; - this.forceBrowserNext = false; - const client = !forceBrowser && offerHasNativeVideoCodec(payload.description.sdp) - ? await this.nativeClient.catch(() => null) + const client = offerHasNativeVideoCodec(payload.description.sdp) + ? await this.acquireNativeClient().catch(() => null) : null; if (this.disposed) return; this.backend = client @@ -104,9 +102,7 @@ export class NativeCapableViewerPeer implements ViewerMediaPeer { client, this.nativeSessionId, this.edgeCapacity, - () => { - this.forceBrowserNext = true; - }, + this.onNativeUnavailable, ) : this.newBrowserPeer(); } @@ -592,11 +588,11 @@ class NativeViewerPeer implements ViewerMediaPeer { void this.client.closeReceiver(this.sessionId, connectionId).catch( () => undefined, ); + this.onFailure(); if (this.recoveryOwner === "route") { this.reportRecoveryExhausted(); return; } - this.onFailure(); if (!this.requestRecovery(true)) { this.reportRecoveryExhausted(); } diff --git a/src/client/native/wire.ts b/src/client/native/wire.ts index 501f24b..e71a393 100644 --- a/src/client/native/wire.ts +++ b/src/client/native/wire.ts @@ -1,9 +1,10 @@ import { z } from "zod"; +import { sfuMediaSchema } from "../../shared/protocol"; -export const NATIVE_CLIENT_PROTOCOL = 8; +export const NATIVE_CLIENT_PROTOCOL = 9; export const NATIVE_CLIENT_PORT_START = 39_721; export const NATIVE_CLIENT_PORT_END = 39_730; -export const NATIVE_CLIENT_SUBPROTOCOL = "screener-client-v8"; +export const NATIVE_CLIENT_SUBPROTOCOL = "screener-client-v9"; const decimalIdentifierSchema = z.string().regex(/^[1-9]\d{0,19}$/); const opaqueIdentifierSchema = z @@ -16,8 +17,15 @@ export const nativeHealthSchema = z .object({ protocol: z.literal(NATIVE_CLIENT_PROTOCOL), service: z.literal("screener-client"), - port: z.number().int().min(NATIVE_CLIENT_PORT_START).max(NATIVE_CLIENT_PORT_END), - instanceToken: z.string().length(43).regex(/^[A-Za-z0-9_-]+$/), + port: z + .number() + .int() + .min(NATIVE_CLIENT_PORT_START) + .max(NATIVE_CLIENT_PORT_END), + instanceToken: z + .string() + .length(43) + .regex(/^[A-Za-z0-9_-]+$/), nativeMedia: z .object({ video: z.boolean(), @@ -105,6 +113,13 @@ export const readyResponseSchema = z export const pongResponseSchema = z .object({ ...responseBase, type: z.literal("pong") }) .strict(); +export const requestFailedResponseSchema = z + .object({ + ...responseBase, + type: z.literal("request-failed"), + code: z.literal("operation-failed"), + }) + .strict(); export const captureOptionsResponseSchema = z .object({ ...responseBase, @@ -157,7 +172,10 @@ export const edgeOfferResponseSchema = z type: z.literal("edge-offer"), shareId: opaqueIdentifierSchema, connectionId: opaqueIdentifierSchema, - sdp: z.string().min(1).max(48 * 1024), + sdp: z + .string() + .min(1) + .max(48 * 1024), }) .strict(); export const receiveAnswerResponseSchema = z @@ -166,12 +184,30 @@ export const receiveAnswerResponseSchema = z type: z.literal("receive-answer"), shareId: opaqueIdentifierSchema, connectionId: opaqueIdentifierSchema, - sdp: z.string().min(1).max(48 * 1024), + sdp: z + .string() + .min(1) + .max(48 * 1024), audio: z.boolean(), codec: nativeVideoCodecSchema, }) .strict(); +export const publicationResponseSchema = z + .object({ + ...responseBase, + type: z.enum(["publication-offer", "publication-media"]), + shareId: opaqueIdentifierSchema, + publicationGeneration: opaqueIdentifierSchema, + connectionId: opaqueIdentifierSchema, + sdp: z + .string() + .max(48 * 1024) + .optional(), + media: sfuMediaSchema, + }) + .strict(); + export const nativeAckResponseSchema = z .object({ ...responseBase, @@ -184,6 +220,10 @@ export const nativeAckResponseSchema = z "receive-stopped", "share-stopped", "share-paused", + "publication-answer-accepted", + "publication-candidate-accepted", + "publication-layers-accepted", + "publication-closed", ]), }) .strict(); @@ -194,6 +234,76 @@ const eventBase = { }; export const nativeEventSchema = z.discriminatedUnion("type", [ + z + .object({ + ...eventBase, + type: z.literal("publication-quality"), + publicationGeneration: opaqueIdentifierSchema, + connectionId: opaqueIdentifierSchema, + sampleTimestampMs: z + .number() + .int() + .nonnegative() + .max(Number.MAX_SAFE_INTEGER), + sampleWindowMs: z.number().int().min(1_000).max(5_000), + rtpStatsId: z.string().min(1).max(256), + trackIdentifier: z.string().min(1).max(256), + codec: nativeVideoCodecSchema, + videoEncodingCount: z.number().int().min(1).max(3), + activeVideoEncodingCount: z.number().int().min(0).max(3), + rid: z.string().max(16), + intervalFramesSent: z + .number() + .int() + .nonnegative() + .max(Number.MAX_SAFE_INTEGER), + framesPerSecond: z.number().finite().nonnegative().max(240), + width: z.number().int().nonnegative().max(16_384), + height: z.number().int().nonnegative().max(16_384), + bitrateKbps: z.number().finite().nonnegative().max(100_000), + audioBitrateKbps: z.number().finite().nonnegative().max(1_000), + availableOutgoingKbps: z + .number() + .finite() + .nonnegative() + .max(100_000) + .nullable(), + state: z.enum(["unknown", "healthy", "degraded"]), + reason: z.enum(["none", "bandwidth"]).nullable(), + }) + .strict() + .refine(({ state, reason }) => + state === "unknown" + ? reason === null + : state === "healthy" + ? reason === "none" + : reason === "bandwidth", + ), + z + .object({ + ...eventBase, + type: z.literal("publication-candidate"), + publicationGeneration: opaqueIdentifierSchema, + connectionId: opaqueIdentifierSchema, + candidate: candidateSchema.nullable(), + }) + .strict(), + z + .object({ + ...eventBase, + type: z.literal("publication-state"), + publicationGeneration: opaqueIdentifierSchema, + connectionId: opaqueIdentifierSchema, + state: z.enum([ + "new", + "connecting", + "connected", + "disconnected", + "failed", + "closed", + ]), + }) + .strict(), z .object({ ...eventBase, @@ -239,13 +349,21 @@ export const nativeEventSchema = z.discriminatedUnion("type", [ ...eventBase, type: z.literal("edge-quality"), connectionId: opaqueIdentifierSchema, - sampleTimestampMs: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), + sampleTimestampMs: z + .number() + .int() + .nonnegative() + .max(Number.MAX_SAFE_INTEGER), sampleWindowMs: z.number().int().min(1_000).max(5_000), rtpStatsId: z.string().min(1).max(256), trackIdentifier: z.string().min(1).max(256), state: z.enum(["unknown", "healthy", "degraded"]), reason: z.enum(["none", "bandwidth"]).nullable(), - intervalFramesEncoded: z.number().int().nonnegative().max(Number.MAX_SAFE_INTEGER), + intervalFramesEncoded: z + .number() + .int() + .nonnegative() + .max(Number.MAX_SAFE_INTEGER), framesPerSecond: z.number().finite().nonnegative().max(240), bitrateKbps: z.number().finite().nonnegative().max(100_000), availableOutgoingKbps: z.number().finite().nonnegative().max(100_000), diff --git a/src/client/pages/HostPage.tsx b/src/client/pages/HostPage.tsx index fd4f154..9db659f 100644 --- a/src/client/pages/HostPage.tsx +++ b/src/client/pages/HostPage.tsx @@ -14,6 +14,7 @@ import { MAX_VIEWER_PASSWORD_LENGTH, viewerPasswordSchema, type CreateRoomResponse, + type ClientMessage, type IceConfig, type ParticipantPresenceEntry, type PreparedRouteCandidate, @@ -91,6 +92,7 @@ import { type HostRoomState, mergeAuthenticatedHostRoom, readHostRoom, + releaseHostRoom, readPreferredRoomId, readViewerGrant, replaceViewerInvite, @@ -125,7 +127,6 @@ import { HostSfuRoute } from "../media/host-sfu-route"; import { HostProvisionalChild, } from "../media/host-provisional-child"; -import { SfuStandbyPrewarmer } from "../media/sfu-standby-prewarmer"; import { invalidateSenderQualityEvidence, senderQualityEvidenceFromSnapshot, @@ -153,6 +154,8 @@ import { NativeSenderPeer, shouldUseBrowserQualityCandidate, } from "../native/native-sender-peer"; +import { NativeSfuPublisher } from "../native/native-sfu-publisher"; +import { SfuPublisher } from "../sfu/publisher"; import { NativeMediaBridge } from "../native/media-bridge"; import { NativeMediaIngress } from "../native/media-ingress"; import { @@ -445,10 +448,22 @@ export function HostPage({ const [nativeSources, setNativeSources] = useState(null); const [details, setDetails] = useState(null); - const [room, setRoom] = useState(() => - hostRoomFromStored(readHostRoom()), - ); + const [room, setRoom] = useState(null); const roomRef = useRef(room); + const roomInitializationRef = useRef>(Promise.resolve()); + useEffect(() => { + let mounted = true; + roomInitializationRef.current = readHostRoom().then((stored) => { + if (!mounted || roomRef.current !== null) return; + const restored = hostRoomFromStored(stored); + roomRef.current = restored; + setRoom(restored); + }); + return () => { + mounted = false; + releaseHostRoom(); + }; + }, []); useEffect(() => { roomRef.current = room; }, [room]); @@ -478,7 +493,7 @@ export function HostPage({ () => readStoredDisplayName() !== null, ); const [displayName, setDisplayName] = useState(() => - readDisplayName(defaultHostDisplayName("", vis)), + readDisplayName(defaultHostDisplayName(vis)), ); const [displayNameDraft, setDisplayNameDraft] = useState(displayName); const [displayNameError, setDisplayNameError] = useState(null); @@ -541,10 +556,7 @@ export function HostPage({ }, [vis]); useEffect(() => { if (hasCustomDisplayName) return; - const fallback = defaultHostDisplayName( - hostClientIdRef.current ?? "", - vis, - ); + const fallback = defaultHostDisplayName(vis); if (displayNameRef.current === fallback) return; displayNameRef.current = fallback; setDisplayName(fallback); @@ -582,7 +594,6 @@ export function HostPage({ const sharingPausedRef = useRef(false); const retiringStreamRef = useRef(null); const hostSfuRouteRef = useRef(null); - const sfuStandbyPrewarmerRef = useRef(null); const nativeClientRef = useRef(null); const nativeClientConnectRef = useRef | null>(null); const nativeShareGenerationRef = useRef(null); @@ -686,6 +697,7 @@ export function HostPage({ () => () => { activeGenerationRef.current = null; generationRef.current += 1; + roomMutationRef.current = null; sourceSwitchRef.current = null; qualityChangeRef.current = null; pendingQualityChangeRef.current = null; @@ -712,7 +724,6 @@ export function HostPage({ activeRouteRevisionRef.current = 0; void hostSfuRouteRef.current?.disconnect(); hostSfuRouteRef.current = null; - sfuStandbyPrewarmerRef.current?.dispose(); nativeSourceRequestRef.current = null; nativeSourcePathRef.current = null; disposeNativeShare(); @@ -784,6 +795,21 @@ export function HostPage({ getStream: () => streamRef.current, getProfile: () => qualitySettingsRef.current, getVideoCodec: () => videoCodecRef.current.primary, + createPublisher: (onDisconnected, onStats) => { + const events = { + send: (message: ClientMessage) => isCurrentGeneration(generation) && hostSfuRouteRef.current === route + ? signalRef.current?.send(message) === true : false, + onDisconnected, onStats, + }; + if (nativeModeRef.current) { + const client = nativeClientRef.current; + const shareId = nativeShareGenerationRef.current; + const ice = iceConfigRef.current; + if (!client || !shareId || !ice) throw new Error("Native publication source is unavailable"); + return new NativeSfuPublisher(client, shareId, ice, events); + } + return new SfuPublisher(events); + }, reconcileChildren: (childPeerIds) => { if ( isCurrentGeneration(generation) && @@ -818,15 +844,6 @@ export function HostPage({ return route; } - function setSfuStandbyUrl(url: string | null | undefined): void { - if (!url) { - sfuStandbyPrewarmerRef.current?.setUrl(null); - return; - } - sfuStandbyPrewarmerRef.current ??= new SfuStandbyPrewarmer(); - sfuStandbyPrewarmerRef.current.setUrl(url); - } - function syncHostSfuQualityWarning( route: HostSfuRoute, generation: number, @@ -846,6 +863,7 @@ export function HostPage({ sourceSwitchRef.current = null; qualityChangeRef.current = null; pendingQualityChangeRef.current = null; + commitQuality(qualitySettingsRef.current); codecProbeAbortRef.current?.abort(); codecProbeAbortRef.current = null; videoCodecRef.current = VP8_ONLY_VIDEO_CODEC; @@ -875,7 +893,6 @@ export function HostPage({ hostPeerIdRef.current = null; void hostSfuRouteRef.current?.disconnect(); hostSfuRouteRef.current = null; - sfuStandbyPrewarmerRef.current?.setUrl(null); streamRef.current?.getTracks().forEach((track) => track.stop()); retiringStreamRef.current?.getTracks().forEach((track) => track.stop()); streamRef.current = null; @@ -909,11 +926,11 @@ export function HostPage({ ); } - function forgetRoom(expected?: HostRoomState): boolean { + function forgetRoom(expected?: HostRoomState, keepResumeHint = false): boolean { if (expected && !isCurrentRoomAuthority(expected)) { return false; } - clearHostRoom(); + clearHostRoom(keepResumeHint); roomRef.current = null; setRoom(null); setCopied(false); @@ -992,7 +1009,12 @@ export function HostPage({ endSharing({ key: "host.roomReplaced" }, false); } replaceViewerInvite(activeRoom.roomId, null); - writeHostRoom(replacement); + if (!(await writeHostRoom( + replacement, + () => roomMutationRef.current === mutation, + ))) { + throw new Error("Host room is already open in another tab"); + } writePreferredRoom(replacement.roomId); roomRef.current = replacement; setRoom(replacement); @@ -1027,7 +1049,12 @@ export function HostPage({ if (replacement.roomId !== activeRoom.roomId) { replaceViewerInvite(activeRoom.roomId, null); } - writeHostRoom(replacement); + if (!(await writeHostRoom( + replacement, + () => roomMutationRef.current === mutation, + ))) { + throw new Error("Host room is already open in another tab"); + } writePreferredRoom(replacement.roomId); roomRef.current = replacement; setRoom(replacement); @@ -1404,7 +1431,7 @@ export function HostPage({ if (!client || !isCurrentShare(generation, shareGeneration)) return; const ingress = new NativeMediaIngress(shareGeneration, client, () => { recoverBrowserFanout(ingress); - }); + }, () => qualitySettingsRef.current); nativeMediaIngressRef.current = ingress; nativeShareGenerationRef.current = shareGeneration; try { @@ -1630,10 +1657,6 @@ export function HostPage({ if (!nativeUpdate && captureChanged) { await applyCaptureProfile(activeStream, nextProfile); } - const ingress = nativeMediaIngressRef.current; - if (ingress && !(await ingress.updateProfile(nextProfile))) { - throw new Error("Native media ingress is unavailable"); - } if ( !isCurrentGeneration(generation) || qualityChangeRef.current !== token || @@ -1649,6 +1672,20 @@ export function HostPage({ setDetails(captureDetails(activeStream)); } signalRef.current?.setHostQualitySettings(appliedProfile); + const ingress = nativeMediaIngressRef.current; + if (ingress) { + const updated = await ingress.updateProfile(appliedProfile).catch(() => false); + if ( + !isCurrentGeneration(generation) || + qualityChangeRef.current !== token || + streamRef.current !== activeStream + ) { + return; + } + if (!updated && nativeMediaIngressRef.current === ingress) { + recoverBrowserFanout(ingress); + } + } const activeSfuRoute = hostSfuRouteRef.current; const [results, sfuUpdated] = await Promise.all([ Promise.all( @@ -1670,7 +1707,8 @@ export function HostPage({ ]); if ( isCurrentGeneration(generation) && - qualityChangeRef.current === token + qualityChangeRef.current === token && + streamRef.current === activeStream ) { const failed = results.filter((updated) => !updated).length; const sfuWarning = @@ -1694,7 +1732,8 @@ export function HostPage({ } catch (error) { if ( isCurrentGeneration(generation) && - qualityChangeRef.current === token + qualityChangeRef.current === token && + streamRef.current === activeStream ) { if (pendingQualityChangeRef.current === null) { advancedQualityRef.current = qualitySettingsRef.current; @@ -2174,9 +2213,6 @@ export function HostPage({ setViewerPasswordEnabled(message.viewerPasswordEnabled); setViewerPasswordDraft(authenticatedProfile.roomPassword ?? ""); setViewerPasswordVisible(false); - setSfuStandbyUrl( - "sfuStandbyUrl" in message ? message.sfuStandbyUrl : null, - ); setMaxViewers(message.maxViewers); routePolicyRef.current = { ...message.routePolicy }; setRoutePolicy({ ...message.routePolicy }); @@ -2192,7 +2228,12 @@ export function HostPage({ pendingQualitySettings ?? message.qualitySettings; activeRouteRevisionRef.current = message.routeRevision; if (reauthenticated) { + const draft = qualityChangeRef.current ? advancedQualityRef.current : null; commitQuality(currentQualitySettings); + if (draft) { + advancedQualityRef.current = draft; + setAdvancedQuality(draft); + } const endpointUpdates = [ ...[...peersRef.current.values()].map((peer) => peer.updateProfile(currentQualitySettings), @@ -2215,8 +2256,8 @@ export function HostPage({ assignment: message.routeAssignment, }) .then(async () => { - if (reauthenticated && hostSfuRouteRef.current === route) { - await route.updateProfile(currentQualitySettings); + if (reauthenticated && isCurrentGeneration(generation) && hostSfuRouteRef.current === route) { + await route.updateProfile(qualitySettingsRef.current); } syncHostSfuQualityWarning(route, generation); }); @@ -2290,6 +2331,10 @@ export function HostPage({ .then(() => syncHostSfuQualityWarning(route, generation)); return; } + if (message.type === "sfu-signal") { + void hostSfuRouteRef.current?.acceptSignal(message); + return; + } if (message.type === "signal") { if ( hostProvisionalChildRef.current?.acceptSignal( @@ -2426,8 +2471,9 @@ export function HostPage({ let createdRoom: HostRoomState | null = null; let claimedRoom = false; try { - const reusableRoom = roomRef.current; - createdRoom = reusableRoom ?? hostRoomFromStored(readHostRoom()); + await roomInitializationRef.current; + if (!isCurrentShare(generation, shareGeneration)) return; + createdRoom = roomRef.current; if (!createdRoom) { const response = await createRoom( creationProfileRef.current.codeEntryPolicy, @@ -2441,7 +2487,12 @@ export function HostPage({ closeAbandonedRoom(createdRoom); return; } - writeHostRoom(createdRoom); + if (!(await writeHostRoom( + createdRoom, + () => isCurrentShare(generation, shareGeneration), + ))) { + throw new Error("Host room is already open in another tab"); + } roomRef.current = createdRoom; setRoom(createdRoom); claimedRoom = true; @@ -2455,7 +2506,7 @@ export function HostPage({ let authenticated = false; const hostClientId = getStableClientId("host", activeRoom.roomId); hostClientIdRef.current = hostClientId; - const hostFallback = defaultHostDisplayName(hostClientId, visRef.current); + const hostFallback = defaultHostDisplayName(visRef.current); const initialDisplayName = readDisplayName(hostFallback); displayNameRef.current = initialDisplayName; setDisplayName(initialDisplayName); @@ -2489,6 +2540,7 @@ export function HostPage({ signalRef.current === signal ) { endSharing({ key: hostTerminationKey(reason) }, false, "warning"); + if (reason === "SESSION_REPLACED") forgetRoom(activeRoom, true); } }, onAccessRequired: () => { @@ -2532,10 +2584,11 @@ export function HostPage({ peersRef.current.forEach((peer) => peer.updateIceConfig(message.iceConfig), ); - writeHostRoom({ - ...activeRoom, - expiresAt: message.roomExpiresAt, - }); + void writeHostRoom( + { ...activeRoom, expiresAt: message.roomExpiresAt }, + () => isCurrentShare(generation, shareGeneration) && + signalRef.current === signal, + ); writePreferredRoom(activeRoom.roomId); setPhase("live"); } @@ -2567,7 +2620,13 @@ export function HostPage({ closeAbandonedRoom(replacement); return; } - writeHostRoom(replacement); + if (!(await writeHostRoom( + replacement, + () => isCurrentShare(generation, shareGeneration), + ))) { + closeAbandonedRoom(replacement); + throw new Error("Host room is already open in another tab"); + } roomRef.current = replacement; setRoom(replacement); const replacementSignal = connectSignal(replacement); @@ -2675,9 +2734,17 @@ export function HostPage({ nativeCaptureDetails(qualitySettingsRef.current, activeStream), ); } + const sfuUpdated = await hostSfuRouteRef.current?.updateProfile(qualitySettingsRef.current) ?? true; + if ( + !isCurrentGeneration(generation) || + sourceSwitchRef.current !== token || + nativeClientRef.current !== client + ) { + return; + } setNotice(sourceSwitchNotice({ failedPeerCount: 0, - sfuReplaced: true, + sfuReplaced: sfuUpdated, sfuWarning: null, })); } catch (error) { @@ -2765,7 +2832,7 @@ export function HostPage({ if (!client) throw new Error("Native media ingress is unavailable"); const replacement = new NativeMediaIngress(ingress.shareId, client, () => { recoverBrowserFanout(replacement); - }); + }, () => qualitySettingsRef.current); try { await replacement.start(captured, qualitySettingsRef.current); if (!isCurrentGeneration(generation) || nativeMediaIngressRef.current !== ingress) { @@ -3084,9 +3151,7 @@ export function HostPage({ } function commitDisplayName(): void { - const hostFallback = hostClientIdRef.current - ? defaultHostDisplayName(hostClientIdRef.current, vis) - : defaultHostDisplayName("", vis); + const hostFallback = defaultHostDisplayName(vis); const saved = saveDisplayName(displayNameDraft, hostFallback); if (!saved) { setDisplayNameError(say("host.nameError")); @@ -4158,7 +4223,7 @@ export function HostPage({ -
+
{(Object.keys(QUALITY_PROFILES) as QualityProfileId[]).map( (id, index) => ( @@ -4210,6 +4275,7 @@ export function HostPage({ "hint-advanced",
readStoredDisplayName() !== null, ); const [displayName, setDisplayName] = useState(() => - readDisplayName(defaultViewerDisplayName(viewerClientId, vis)), + readDisplayName(defaultViewerDisplayName(vis)), ); const [displayNameDraft, setDisplayNameDraft] = useState(displayName); const [displayNameError, setDisplayNameError] = useState(false); @@ -423,7 +422,7 @@ export function ViewerPage({ const displayNameRef = useRef(displayName); useEffect(() => { if (hasCustomDisplayName) return; - const fallback = defaultViewerDisplayName(viewerClientId, vis); + const fallback = defaultViewerDisplayName(vis); if (displayNameRef.current === fallback) return; displayNameRef.current = fallback; setDisplayName(fallback); @@ -625,7 +624,6 @@ export function ViewerPage({ let pendingPeer: PendingPeerRoute | null = null; const decodedFrameStall = new DecodedFrameStallDetector(); const messageAuthority = new ViewerMessageAuthority(); - let sfuStandbyPrewarmer: SfuStandbyPrewarmer | null = null; const relayChildEvidenceCurrent = new Map< string, ViewerQualityEvidencePresentation @@ -633,12 +631,14 @@ export function ViewerPage({ const relayChildEvidenceTimers = new Map(); let sfuTransportConnected = false; let nativeClientPromise: Promise | null = null; + let nativeViewerAvailable = true; const nativeViewerSessionId = createOpaqueId(); - const acquireNativeClient = (): Promise => { - if (!launchedByClient) return Promise.resolve(null); + const acquireNativeClient = async (): Promise => { + if (!launchedByClient || !nativeViewerAvailable) return null; nativeClientPromise ??= NativeClient.connect(); - return nativeClientPromise; + const client = await nativeClientPromise; + return nativeViewerAvailable ? client : null; }; function createViewerMediaPeer( @@ -653,7 +653,8 @@ export function ViewerPage({ iceConfig, events, options, - acquireNativeClient(), + acquireNativeClient, + () => { nativeViewerAvailable = false; }, nativeViewerSessionId, MAX_ENDPOINT_MEDIA_CHILDREN, ); @@ -699,15 +700,6 @@ export function ViewerPage({ window.addEventListener("pagehide", suspendForPageLifecycle); window.addEventListener("pageshow", recoverFromPageLifecycle); - function setSfuStandbyUrl(url: string | null | undefined): void { - if (!url) { - sfuStandbyPrewarmer?.setUrl(null); - return; - } - sfuStandbyPrewarmer ??= new SfuStandbyPrewarmer(); - sfuStandbyPrewarmer.setUrl(url); - } - const signal = new SignalingClient( { roomId, @@ -732,7 +724,6 @@ export function ViewerPage({ return; } messageAuthority.invalidate(); - setSfuStandbyUrl(null); setAssignedRoute(null); clearViewerSfuRoute(); clearPeerState(true); @@ -746,7 +737,6 @@ export function ViewerPage({ onAccessRequired: () => { if (active) { messageAuthority.invalidate(); - setSfuStandbyUrl(null); setAssignedRoute(null); clearViewerSfuRoute(); clearPeerState(true); @@ -956,6 +946,7 @@ export function ViewerPage({ source.codec, { connectionId: source.connectionId, + getProfile: () => currentQualitySettings, format: () => { const settings = remoteMediaRef.current?.stream .getVideoTracks()[0] @@ -1716,9 +1707,6 @@ export function ViewerPage({ viewerAuthorizationGeneration = message.viewerAuthorizationGeneration; currentRoutePolicy = message.routePolicy; - setSfuStandbyUrl( - "sfuStandbyUrl" in message ? message.sfuStandbyUrl : null, - ); const nextRouteRevision = message.routeRevision; if (nextRouteRevision !== currentRouteRevision) { clearRelayChildEvidence(); @@ -1873,6 +1861,10 @@ export function ViewerPage({ await ensureViewerSfuRoute().acceptConfig(message); return; } + if (message.type === "sfu-signal") { + await viewerSfuRoute?.acceptSignal(message); + return; + } if (message.type === "quality-settings") { currentQualitySettings = message.qualitySettings; void viewerRelay?.updateProfile(currentQualitySettings); @@ -1885,7 +1877,6 @@ export function ViewerPage({ ) { currentShareGeneration = message.shareGeneration; currentRoutePolicy = message.routePolicy; - setSfuStandbyUrl(message.sfuStandbyUrl ?? null); } return; } @@ -1988,7 +1979,6 @@ export function ViewerPage({ pendingRouteConnection = null; currentShareGeneration = null; currentRoutePolicy = DEFAULT_ROUTE_POLICY; - setSfuStandbyUrl(null); setAssignedRoute(null); currentHostOnline = false; currentHostPaused = false; @@ -2009,7 +1999,6 @@ export function ViewerPage({ } viewerAuthorizationGeneration = null; clearViewerGrant(roomId); - setSfuStandbyUrl(null); setAssignedRoute(null); clearViewerSfuRoute(); clearPeerState(true); @@ -2023,7 +2012,6 @@ export function ViewerPage({ return; } if (message.type === "room-closed") { - setSfuStandbyUrl(null); setAssignedRoute(null); clearViewerSfuRoute(); clearPeerState(true); @@ -2048,7 +2036,6 @@ export function ViewerPage({ "ROOM_FULL", ].includes(message.code) ) { - setSfuStandbyUrl(null); setAssignedRoute(null); clearViewerSfuRoute(); clearPeerState(true); @@ -2095,7 +2082,6 @@ export function ViewerPage({ qualityEvidenceReporterRef.current = null; } clearRelayChildEvidence(); - sfuStandbyPrewarmer?.dispose(); signal.stop(); clearParticipantPresence(); if (signalRef.current === signal) { @@ -2218,7 +2204,7 @@ export function ViewerPage({ } function commitDisplayName(): void { - const fallback = defaultViewerDisplayName(viewerClientId, vis); + const fallback = defaultViewerDisplayName(vis); const saved = saveDisplayName(displayNameDraft, fallback); if (!saved) { setDisplayNameError(true); diff --git a/src/client/pages/host-page-notices.ts b/src/client/pages/host-page-notices.ts index 8f78abb..72afd1b 100644 --- a/src/client/pages/host-page-notices.ts +++ b/src/client/pages/host-page-notices.ts @@ -1,4 +1,5 @@ import type { ServerMessage } from "../../shared/protocol"; +import { NativeMediaBridgeError } from "../native/media-bridge"; import { joinSentences, say, type CopyKey } from "../ui/copy"; export type HostAction = @@ -39,6 +40,7 @@ export function hostActionErrorNotice( error: unknown, action: HostAction, ): string { + if (error instanceof NativeMediaBridgeError) return say(HOST_ACTION_FALLBACK.connection); if ( error instanceof DOMException && (action === "capture" || action === "source") diff --git a/src/client/sfu/connection-options.ts b/src/client/sfu/connection-options.ts deleted file mode 100644 index 697e978..0000000 --- a/src/client/sfu/connection-options.ts +++ /dev/null @@ -1,8 +0,0 @@ -import type { RoomConnectOptions } from "livekit-client"; - -export function sfuRoomConnectOptions(): RoomConnectOptions { - return { - autoSubscribe: false, - rtcConfig: { iceServers: [] }, - }; -} diff --git a/src/client/sfu/peer.ts b/src/client/sfu/peer.ts new file mode 100644 index 0000000..e5fdd9d --- /dev/null +++ b/src/client/sfu/peer.ts @@ -0,0 +1,157 @@ +import type { + ServerMessage, + SfuMedia, + SfuSignalMessage, +} from "../../shared/protocol"; + +export type SfuConnectionConfig = Omit< + Extract, + "type" +>; + +interface SfuPeerEvents { + send: (message: SfuSignalMessage) => boolean; + onState: (state: RTCPeerConnectionState) => void; + onTrack?: (event: RTCTrackEvent) => void; +} + +// The room WebSocket owns signaling; losing it does not retire healthy media. +export class SfuPeer { + readonly pc = new RTCPeerConnection({ iceServers: [] }); + private localCandidates: NonNullable[] = []; + private remoteCandidates: RTCIceCandidateInit[] = []; + private descriptionSent = false; + private pendingDescription: Pick< + SfuSignalMessage, + "kind" | "description" | "media" + > | null = null; + private closed = false; + private signalTail: Promise = Promise.resolve(); + + constructor( + private config: SfuConnectionConfig, + private readonly events: SfuPeerEvents, + ) { + this.pc.onicecandidate = ({ candidate }) => { + if (this.closed || !candidate || candidate.protocol === "tcp") return; + const value = { ...candidate.toJSON(), candidate: candidate.candidate }; + if (this.localCandidates.length >= 64) this.events.onState("failed"); + else { + this.localCandidates.push(value); + this.flushSignaling(); + } + }; + this.pc.onconnectionstatechange = () => { + if (!this.closed) this.events.onState(this.pc.connectionState); + }; + this.pc.ontrack = (event) => { + if (!this.closed) this.events.onTrack?.(event); + }; + } + + updateConfig(config: SfuConnectionConfig): void { + if ( + this.config.connectionId === config.connectionId && + this.config.publicationGeneration === config.publicationGeneration + ) { + this.config = config; + this.flushSignaling(); + } + } + + send( + payload: Pick< + SfuSignalMessage, + "kind" | "description" | "candidate" | "media" + >, + ): boolean { + return ( + !this.closed && + this.events.send({ + type: "sfu-signal", + revision: this.config.revision, + publicationGeneration: this.config.publicationGeneration, + connectionId: this.config.connectionId, + ...payload, + }) + ); + } + + async sendDescription( + description: RTCSessionDescriptionInit, + media?: SfuMedia, + ): Promise { + this.descriptionSent = false; + await this.pc.setLocalDescription(description); + if (this.closed) return; + const local = this.pc.localDescription; + if (!local || (local.type !== "offer" && local.type !== "answer")) { + throw new Error("SFU has no local description"); + } + this.pendingDescription = { + kind: "description", + description: { type: local.type, sdp: local.sdp }, + ...(media ? { media } : {}), + }; + this.flushSignaling(); + } + + private flushSignaling(): void { + if (this.closed) return; + if (this.pendingDescription) { + if (!this.send(this.pendingDescription)) return; + this.pendingDescription = null; + this.descriptionSent = true; + } + if (!this.descriptionSent) return; + while (this.localCandidates[0]) { + if (!this.send({ kind: "candidate", candidate: this.localCandidates[0] })) + return; + this.localCandidates.shift(); + } + } + + acceptSignal( + message: SfuSignalMessage, + onDescription: (description: RTCSessionDescriptionInit) => Promise, + onLayers?: (activeCount: number) => Promise, + ): Promise { + if ( + this.closed || + message.connectionId !== this.config.connectionId || + message.publicationGeneration !== this.config.publicationGeneration + ) + return Promise.resolve(); + const work = async (): Promise => { + if (this.closed) return; + if (message.kind === "layers" && message.activeCount !== undefined) { + await onLayers?.(message.activeCount); + } else if (message.kind === "description" && message.description) { + await onDescription(message.description); + for (const candidate of this.remoteCandidates.splice(0)) + await this.pc.addIceCandidate(candidate); + } else if (message.kind === "candidate" && message.candidate) { + if (this.pc.remoteDescription) + await this.pc.addIceCandidate(message.candidate); + else if (this.remoteCandidates.length < 64) + this.remoteCandidates.push(message.candidate); + else throw new Error("Too many SFU ICE candidates"); + } + }; + const next = this.signalTail.then(work); + this.signalTail = next.catch(() => undefined); + return next; + } + + close(): void { + if (this.closed) return; + this.closed = true; + this.pc.onicecandidate = null; + this.pc.onconnectionstatechange = null; + this.pc.ontrack = null; + this.localCandidates = []; + this.remoteCandidates = []; + this.pendingDescription = null; + this.pc.close(); + } +} diff --git a/src/client/sfu/publisher.ts b/src/client/sfu/publisher.ts index 70849cc..5812726 100644 --- a/src/client/sfu/publisher.ts +++ b/src/client/sfu/publisher.ts @@ -1,11 +1,5 @@ +import type { SfuMedia, SfuSignalMessage } from "../../shared/protocol"; import { say } from "../ui/copy"; -import type { - LocalTrackPublication, - Room, - Track, - TrackPublishOptions, -} from "livekit-client"; - import { audioSenderParameterWarning, applyVideoCaptureProfile, @@ -13,8 +7,7 @@ import { configureScreenAudioSender, configureVideoSender, needsStartupVideoProfile, - resolveScreenAudioQuality, - screenAudioQualityEqual, + QUALITY_RESOLUTIONS, screenAudioBitrate, senderParameterWarning, STARTUP_VIDEO_ENCODED_FRAMES, @@ -22,11 +15,14 @@ import { videoQualitySettingsEqual, type AudioSenderParameterReadback, type QualityProfile, - type ScreenAudioQuality, type VideoSenderParameterReadback, } from "../media/quality"; import type { ConnectionMetrics } from "../types"; -import type { BrowserVideoCodec } from "../webrtc/video-codec"; +import { + applyVideoCodecPreference, + manualVideoCodecPreference, + type BrowserVideoCodec, +} from "../webrtc/video-codec"; import { captureMetrics, collectConnectionMetricsFromReport, @@ -35,18 +31,13 @@ import { createStatsAccumulator, highestActiveVideoRid, maxEncodedVideoFrames, - mergeStatsReports, - type StatsAccumulator, - type NativeSenderQualityAccumulator, } from "../webrtc/stats"; -import { sfuRoomConnectOptions } from "./connection-options"; +import { SfuPeer, type SfuConnectionConfig } from "./peer"; -export interface SfuConnectionConfig { - url: string; - token: string; -} +export type { SfuConnectionConfig } from "./peer"; interface PublisherEvents { + send: (message: SfuSignalMessage) => boolean; onDisconnected?: () => void; onStats?: (metrics: ConnectionMetrics | null) => void; } @@ -59,1429 +50,519 @@ export type SfuPublisherFailureStage = | "audio-publish" | "transport"; -interface PublishedTrack { - publication: LocalTrackPublication; - sourceTrack: MediaStreamTrack; - rawTrack: MediaStreamTrack; - stopOnRelease: boolean; -} - -interface PublishedVideoConfiguration { - readback: VideoSenderParameterReadback; - warning: string | null; -} - -interface PublishedAudioConfiguration { - sender: RTCRtpSender; - readback: AudioSenderParameterReadback; - warning: string | null; -} - -interface PublisherStatsIdentity { - video: PublishedTrack; - videoSender: RTCRtpSender; - videoTrack: MediaStreamTrack; - audio: PublishedTrack | null; - audioSender: RTCRtpSender | null; - audioTrack: MediaStreamTrack | null; - accumulator: StatsAccumulator; - nativeQualityAccumulator: NativeSenderQualityAccumulator; -} - -type LiveKit = typeof import("livekit-client"); -type PublisherState = - | "idle" - | "connecting" - | "prepared" - | "active" - | "disconnected"; - -const roomDisconnects = new WeakMap>(); -const STATS_INTERVAL_MS = 2_000; - export class SfuPublisher { - private room: Room | null = null; - private sdk: LiveKit | null = null; - private video: PublishedTrack | null = null; - private audio: PublishedTrack | null = null; - private desiredProfile: QualityProfile | null = null; - private appliedVideoProfile: QualityProfile | null = null; - private profileRevision = 0; + private peer: SfuPeer | null = null; + private video: MediaStreamTrack | null = null; + private audio: MediaStreamTrack | null = null; + private videoSender: RTCRtpSender | null = null; + private audioSender: RTCRtpSender | null = null; + private profile: QualityProfile | null = null; + private codec: BrowserVideoCodec = "vp8"; + private layerCount = 2; + private activeCount = 2; + private paused = false; + private closed = false; + private readonly ownedTracks = new Set(); + private startupPending = false; + private operationTail: Promise = Promise.resolve(); + private failureStage: SfuPublisherFailureStage | null = null; private senderParameters: VideoSenderParameterReadback | null = null; private audioSenderParameters: AudioSenderParameterReadback | null = null; - private audioParametersSender: RTCRtpSender | null = null; - private videoQualityWarning: string | null = null; - private audioQualityWarning: string | null = null; - private failureStage: SfuPublisherFailureStage | null = null; - private state: PublisherState = "idle"; - private generation = 0; - private operationTail: Promise = Promise.resolve(); - private terminalNotified = false; - private statsVideo: PublishedTrack | null = null; - private statsIdentity: PublisherStatsIdentity | null = null; - private statsInFlight: PublisherStatsIdentity | null = null; + private videoWarning: string | null = null; + private audioWarning: string | null = null; + private stats = createStatsAccumulator(); + private nativeStats = createNativeSenderQualityAccumulator(); + private statsInFlight: typeof this.stats | null = null; private statsTimer: ReturnType | null = null; - private startupVideoProfilePending = false; - private paused = false; - private replacementVideoTrack: MediaStreamTrack | null = null; - private replacementAudioTrack: MediaStreamTrack | null = null; - constructor(private readonly events: PublisherEvents = {}) {} + constructor(private readonly events: PublisherEvents) {} async connect(config: SfuConnectionConfig): Promise { - if (this.state !== "idle") { + if (this.peer || this.closed) throw new Error("SFU publisher cannot be connected twice"); - } + this.peer = new SfuPeer(config, { + send: this.events.send, + onState: (state) => { + if (state === "failed") this.fail("transport"); + else if (state === "disconnected") void this.restartIce(); + }, + }); + return true; + } - const generation = ++this.generation; - this.failureStage = null; - this.state = "connecting"; + updateConfig(config: SfuConnectionConfig): void { + this.peer?.updateConfig(config); + } + async acceptSignal(message: SfuSignalMessage): Promise { + const peer = this.peer; + if (!peer) return; try { - const sdk = await import("livekit-client"); - if (!this.ownsGeneration(generation)) { - return false; - } - - const room = new sdk.Room({ - disconnectOnPageLeave: false, - dynacast: true, - stopLocalTrackOnUnpublish: false, - }); - this.room = room; - this.sdk = sdk; - room.on(sdk.RoomEvent.Disconnected, () => { - if (this.owns(room, generation)) { - const notify = this.state !== "connecting"; - this.failureStage = "transport"; - this.invalidate(); - if (notify) { - this.notifyTerminalDisconnect(); - } - } - }); - room.on(sdk.RoomEvent.Reconnected, () => { - if (this.owns(room, generation) && this.state === "active") { - this.reapplyAudioAfterReconnect(room, generation); - } - }); - - await room.connect(config.url, config.token, sfuRoomConnectOptions()); - if (!this.owns(room, generation)) { - await safeDisconnect(room); - return false; - } - - this.state = "prepared"; - return true; - } catch (error) { - if (!this.ownsGeneration(generation)) { - return false; - } - this.failureStage = "connect"; - const room = this.invalidate(); - if (room) { - await safeDisconnect(room); - } - throw error; + await peer.acceptSignal( + message, + async (description) => { + if (description.type !== "answer") + throw new Error("SFU publisher expected an answer"); + await peer.pc.setRemoteDescription(description); + }, + async (activeCount) => { + await this.enqueue(async () => { + const sender = this.videoSender; + if (this.peer !== peer || !sender) return false; + if (activeCount > this.layerCount) + throw new Error("SFU requested an unavailable encoding"); + this.activeCount = activeCount; + const parameters = sender.getParameters(); + parameters.encodings.forEach((encoding, index) => { + encoding.active = index < activeCount; + }); + await sender.setParameters(parameters); + return this.peer === peer; + }); + }, + ); + } catch { + if (this.peer === peer) this.fail("transport"); } } activate( stream: MediaStream, profile: QualityProfile, - videoCodec: BrowserVideoCodec = "vp8", + codec: BrowserVideoCodec = "vp8", ): Promise { - this.desiredProfile = profile; - this.startupVideoProfilePending = needsStartupVideoProfile(profile); - const effectiveVideoProfile = startupVideoProfile(profile); - ++this.profileRevision; - const generation = this.generation; return this.enqueue(async () => { - const room = this.requireOwnedRoom(generation, "prepared"); - if (!room) { - return false; - } - let failureStage: SfuPublisherFailureStage = "source"; + const peer = this.peer; + if (!peer || this.video) return false; + this.profile = profile; + this.codec = codec; + this.startupPending = needsStartupVideoProfile(profile); + let video: MediaStreamTrack | null = null; try { - const sdk = this.sdk; - if (!sdk) { - throw new Error("SFU publisher SDK is unavailable"); + this.failureStage = "source"; + video = this.ownTrack(cloneSenderVideoTrack(requiredVideo(stream))); + await applyVideoCaptureProfile(video, profile); + if (this.peer !== peer) { + this.releaseTrack(video); + return false; } - - const videoTrack = requiredVideoTrack(stream); - const audioTrack = stream.getAudioTracks()[0] ?? null; - - failureStage = "video-publish"; - const video = await publishVideoTrack( - room, - videoTrack, - profile, - sdk.Track.Source.ScreenShare, - videoPublishOptions(effectiveVideoProfile, videoCodec), + this.video = video; + this.audio = this.ownTrack(stream.getAudioTracks()[0]?.clone() ?? null); + this.setPaused(this.paused); + const dimensions = video.getSettings(); + this.layerCount = + Math.max(dimensions.width ?? 0, dimensions.height ?? 0) >= 480 + ? 2 + : 1; + const videoTransceiver = peer.pc.addTransceiver(video, { + direction: "sendonly", + streams: [stream], + sendEncodings: this.encodings(profile), + }); + this.failureStage = "video-publish"; + if ( + !applyVideoCodecPreference( + videoTransceiver, + manualVideoCodecPreference(codec), + ) + ) { + throw new Error("SFU video codec is unavailable"); + } + this.videoSender = videoTransceiver.sender; + const audioTransceiver = peer.pc.addTransceiver(this.audio ?? "audio", { + direction: "sendonly", + streams: [stream], + sendEncodings: [ + { maxBitrate: screenAudioBitrate(profile.screenAudioQuality) }, + ], + }); + const opus = RTCRtpSender.getCapabilities("audio")?.codecs.filter( + ({ mimeType }) => mimeType.toLowerCase() === "audio/opus", ); - let adoptedVideo = false; - try { - if (!this.owns(room, generation)) { - return false; - } - failureStage = "sender-config"; - const videoConfiguration = await configurePublishedVideo( - video, - effectiveVideoProfile, - () => this.owns(room, generation), - ); - if (!videoConfiguration || !this.owns(room, generation)) { - return false; - } - - let audio: PublishedTrack | null = null; - let audioConfiguration: PublishedAudioConfiguration | null = null; - let audioWarning: string | null = null; - if (audioTrack) { - failureStage = "audio-publish"; - audio = await publishTrack( - room, - audioTrack, - sdk.Track.Source.ScreenShareAudio, - audioPublishOptions(sdk, profile.screenAudioQuality), - ); - if (!this.owns(room, generation)) { - return false; - } - retainPublishedAudioOptions(audio, profile, sdk); - try { - const audioSender = publishedAudioSender(audio); - audioConfiguration = await configurePublishedAudio( - audio, - audioSender, - profile, - () => this.owns(room, generation), - ); - if (!audioConfiguration || !this.owns(room, generation)) { - return false; - } - audioWarning = audioConfiguration.warning; - } catch (error) { - if (!this.owns(room, generation)) { - return false; - } - audioWarning = audioSenderFailureWarning(error); - } - } - - this.video = video; - this.audio = audio; - adoptedVideo = true; - this.appliedVideoProfile = effectiveVideoProfile; - this.retainSenderParameters( - videoConfiguration, - audioConfiguration?.readback ?? null, - audioWarning, - audioConfiguration?.sender ?? null, - ); - this.failureStage = null; - this.state = "active"; - this.startStats(video); - return true; - } finally { - if (!adoptedVideo) { - releasePublishedTrack(video); - } - } + if (!opus?.length) throw new Error("SFU Opus codec is unavailable"); + audioTransceiver.setCodecPreferences(opus); + this.audioSender = audioTransceiver.sender; + this.failureStage = "sender-config"; + await this.configure(profile); + if (this.peer !== peer) return false; + await peer.sendDescription( + await peer.pc.createOffer(), + this.media(profile), + ); + if (this.peer !== peer) return false; + this.failureStage = null; + this.startStats(); + return true; } catch (error) { - if (this.owns(room, generation)) { - this.failureStage = failureStage; - await this.failClosed(room, generation); - } + if (video && this.video !== video) this.releaseTrack(video); + if (this.peer === peer) this.fail(this.failureStage ?? "connect"); throw error; } }); } - deactivate(): Promise { - const generation = this.generation; - return this.enqueue(async () => { - const room = this.requireOwnedRoom(generation, ["prepared", "active"]); - if (!room) { - return false; - } - if (this.state === "prepared") { - return true; - } - this.stopStats(); - - const video = this.video; - const audio = this.audio; - try { - if (audio) { - await unpublishTrack(room, audio); - if (!this.owns(room, generation)) { - return false; - } - } - if (video) { - await unpublishTrack(room, video); - if (!this.owns(room, generation)) { - return false; - } - } - } catch (error) { - if (this.owns(room, generation)) { - await this.failClosed(room, generation); - } - throw error; - } - - this.video = null; - this.audio = null; - this.desiredProfile = null; - this.appliedVideoProfile = null; - this.startupVideoProfilePending = false; - ++this.profileRevision; - this.senderParameters = null; - this.audioSenderParameters = null; - this.audioParametersSender = null; - this.videoQualityWarning = null; - this.audioQualityWarning = null; - this.state = "prepared"; - return true; - }); + async deactivate(): Promise { + await this.disconnect(); + return true; } replaceStream(stream: MediaStream): Promise { - const generation = this.generation; return this.enqueue(async () => { - const room = this.requireOwnedRoom(generation, "active"); - if (!room) { - return false; - } - const sdk = this.sdk; + const peer = this.peer; + const profile = this.profile; const previousVideo = this.video; - const profile = this.desiredProfile; - const previousVideoProfile = this.appliedVideoProfile; - if (!sdk || !previousVideo || !profile || !previousVideoProfile) { - throw new Error("SFU publisher has no active video publication"); - } - const effectiveVideoProfile = startupVideoProfile(profile); - - const nextVideoSource = requiredVideoTrack(stream); - const nextVideoTrack = cloneSenderVideoTrack(nextVideoSource); - try { - await applyVideoCaptureProfile(nextVideoTrack, profile); - } catch { - nextVideoTrack.stop(); - this.failureStage = "source"; - return false; - } - const nextAudioTrack = stream.getAudioTracks()[0] ?? null; - this.replacementVideoTrack = nextVideoTrack; - this.replacementAudioTrack = nextAudioTrack; - this.applyPausedState(nextVideoTrack, nextAudioTrack); const previousAudio = this.audio; - const previousVideoTrack = previousVideo.rawTrack; - let retainedNextVideoTrack = false; - let videoReplaceAttempted = false; - let audioReplaceAttempted = false; - let audioConfiguration: PublishedAudioConfiguration | null = null; - let audioWarning: string | null = null; - this.stopStats(); - + const videoSender = this.videoSender; + const audioSender = this.audioSender; + if (!peer || !profile || !previousVideo || !videoSender || !audioSender) + return false; + const nextVideo = this.ownTrack( + cloneSenderVideoTrack(requiredVideo(stream)), + ); + const nextAudio = this.ownTrack( + stream.getAudioTracks()[0]?.clone() ?? null, + ); + let retained = false; + this.resetStats(); try { - videoReplaceAttempted = true; - await replacePublishedTrack(previousVideo, nextVideoTrack); - if (!this.owns(room, generation)) { - return false; + await applyVideoCaptureProfile(nextVideo, profile); + if (this.peer !== peer) return false; + nextVideo.enabled = !this.paused; + if (nextAudio) { + nextAudio.enabled = !this.paused; + nextAudio.contentHint = "music"; } - - if (previousAudio && nextAudioTrack) { - audioReplaceAttempted = true; - await replacePublishedTrack(previousAudio, nextAudioTrack); - if (!this.owns(room, generation)) { - return false; - } - } else if (previousAudio) { - await unpublishTrack(room, previousAudio); - if (!this.owns(room, generation)) { - return false; - } - this.audio = null; - } else if (nextAudioTrack) { - const nextAudio = await publishTrack( - room, - nextAudioTrack, - sdk.Track.Source.ScreenShareAudio, - audioPublishOptions(sdk, profile.screenAudioQuality), - ); - if (!this.owns(room, generation)) { - return false; - } - this.audio = nextAudio; - } - - const currentAudio = this.audio; - if (currentAudio && nextAudioTrack) { - retainPublishedAudioOptions(currentAudio, profile, sdk); - try { - const audioSender = publishedAudioSender(currentAudio); - audioConfiguration = await configurePublishedAudio( - currentAudio, - audioSender, - profile, - () => this.owns(room, generation), - ); - if (!audioConfiguration || !this.owns(room, generation)) { - return false; - } - audioWarning = audioConfiguration.warning; - } catch (error) { - if (!this.owns(room, generation)) { - return false; - } - audioWarning = audioSenderFailureWarning(error); - } - } - - const videoConfiguration = await configurePublishedVideo( - previousVideo, - effectiveVideoProfile, - () => this.owns(room, generation), - ); - if (!videoConfiguration || !this.owns(room, generation)) { - return false; - } - this.applyPausedState(nextVideoTrack, nextAudioTrack); - previousVideo.rawTrack = nextVideoTrack; - previousVideo.sourceTrack = nextVideoSource; - retainedNextVideoTrack = true; - previousVideoTrack.stop(); - if (previousAudio && nextAudioTrack) { - previousAudio.rawTrack = nextAudioTrack; - } - this.startupVideoProfilePending = needsStartupVideoProfile(profile); - this.appliedVideoProfile = effectiveVideoProfile; - this.retainSenderParameters( - videoConfiguration, - nextAudioTrack - ? (audioConfiguration?.readback ?? this.audioSenderParameters) - : null, - nextAudioTrack ? audioWarning : null, - nextAudioTrack - ? (audioConfiguration?.sender ?? this.audioParametersSender) - : null, - ); - this.startStats(previousVideo); + await videoSender.replaceTrack(nextVideo); + await audioSender.replaceTrack(nextAudio); + if (this.peer !== peer) return false; + this.video = nextVideo; + this.audio = nextAudio; + this.startupPending = needsStartupVideoProfile(profile); + await this.configure(profile); + if (this.peer !== peer) return false; + if (!peer.send({ kind: "media", media: this.media(profile) })) + throw new Error("SFU signaling is unavailable"); + retained = true; + this.releaseTrack(previousVideo); + this.releaseTrack(previousAudio); + this.setPaused(this.paused); return true; - } catch (error) { - if (!this.owns(room, generation)) { - return false; - } - this.failureStage = "source"; - const failureWarning = say("host.fail.sfuSwitch"); - - // A rejected publish/unpublish may have changed server state without - // returning enough ownership information to undo it safely. - if ( - (previousAudio === null && nextAudioTrack !== null) || - (previousAudio !== null && nextAudioTrack === null) - ) { - await this.failClosed(room, generation); - throw error; - } - + } catch { + if (this.peer !== peer) return false; + this.video = previousVideo; + this.audio = previousAudio; try { - if (audioReplaceAttempted && previousAudio) { - await replacePublishedTrack(previousAudio, previousAudio.rawTrack); - if (!this.owns(room, generation)) { - return false; - } - } - if (videoReplaceAttempted) { - await replacePublishedTrack(previousVideo, previousVideoTrack); - if (!this.owns(room, generation)) { - return false; - } - const videoConfiguration = await configurePublishedVideo( - previousVideo, - previousVideoProfile, - () => this.owns(room, generation), - ); - if (!videoConfiguration || !this.owns(room, generation)) { - return false; - } - this.appliedVideoProfile = previousVideoProfile; - this.senderParameters = videoConfiguration.readback; - this.videoQualityWarning = mergeQualityWarnings( - failureWarning, - videoConfiguration.warning, - ); - this.startStats(previousVideo); - } + await videoSender.replaceTrack(previousVideo); + await audioSender.replaceTrack(previousAudio); + await this.configure(profile); + this.videoWarning = say("host.fail.sfuSwitch"); + return false; + } catch { + this.fail("source"); return false; - } catch (rollbackError) { - if (this.owns(room, generation)) { - await this.failClosed(room, generation); - } - throw rollbackError; } } finally { - if (!retainedNextVideoTrack) { - nextVideoTrack.stop(); - } - if (this.replacementVideoTrack === nextVideoTrack) { - this.replacementVideoTrack = null; - this.replacementAudioTrack = null; + if (!retained) { + this.releaseTrack(nextVideo); + this.releaseTrack(nextAudio); } } }); } updateProfile(profile: QualityProfile): Promise { - const previousDesiredProfile = this.desiredProfile; - if (!needsStartupVideoProfile(profile)) { - this.startupVideoProfilePending = false; - } - const effectiveVideoProfile = this.startupVideoProfilePending - ? startupVideoProfile(profile) - : profile; - const requestedVideo = - previousDesiredProfile === null || - !videoQualitySettingsEqual(previousDesiredProfile, profile) || - this.appliedVideoProfile === null || - !videoQualitySettingsEqual( - this.appliedVideoProfile, - effectiveVideoProfile, - ); - const requestedAudio = - previousDesiredProfile === null || - !screenAudioQualityEqual(previousDesiredProfile, profile) || - (this.audio !== null && - (this.audio.publication.audioTrack?.sender !== - this.audioParametersSender || - this.audioSenderParameters?.requestedMaxBitrate !== - screenAudioBitrate(profile.screenAudioQuality))); - this.desiredProfile = profile; - const requestedRevision = ++this.profileRevision; - if (this.state === "active" && this.audio && this.sdk) { - retainPublishedAudioOptions(this.audio, profile, this.sdk); - } - const generation = this.generation; return this.enqueue(async () => { - const room = this.requireOwnedRoom(generation, "active"); - if (!room || requestedRevision !== this.profileRevision) { - return false; - } + const peer = this.peer; const video = this.video; - const audio = this.audio; - const previousVideoProfile = this.appliedVideoProfile; - const sdk = this.sdk; - if (!video || !previousVideoProfile || !sdk) { - throw new Error("SFU publisher has no active video publication"); - } - const updateVideo = - requestedVideo || - !videoQualitySettingsEqual( - previousVideoProfile, - effectiveVideoProfile, - ); - const currentAudioSender = audio?.publication.audioTrack?.sender ?? null; - const updateAudio = - audio !== null && - (requestedAudio || - currentAudioSender !== this.audioParametersSender || - this.audioSenderParameters?.requestedMaxBitrate !== - screenAudioBitrate(profile.screenAudioQuality)); - if (!updateVideo && !updateAudio) { - return true; - } - if (audio) { - retainPublishedAudioOptions(audio, profile, sdk); - } - if (updateVideo) { - this.stopStats(); - } - - let senderParameters = this.senderParameters; - let videoWarning = this.videoQualityWarning; - let appliedVideoProfile = previousVideoProfile; - let videoSucceeded = true; - if (updateVideo) { + const previous = this.profile; + if (!peer || !video || !previous) return false; + if (!needsStartupVideoProfile(profile)) this.startupPending = false; + const videoChanged = !videoQualitySettingsEqual(previous, profile); + const configureVideo = + videoChanged || + this.senderParameters?.applied.degradationPreference !== + (this.startupPending ? startupVideoProfile(profile) : profile) + .degradationPreference; + if (configureVideo) this.resetStats(); + try { + if (videoChanged) await applyVideoCaptureProfile(video, profile); + const result = await this.configure(profile, configureVideo); + if (this.peer !== peer) return false; + if (!peer.send({ kind: "media", media: this.media(profile) })) + throw new Error("SFU signaling is unavailable"); + this.profile = profile; + return result; + } catch { + if (this.peer !== peer) return false; try { - await applyVideoCaptureProfile(video.rawTrack, profile); - const configured = await configurePublishedVideo( - video, - effectiveVideoProfile, - () => - this.owns(room, generation) && - requestedRevision === this.profileRevision && - this.video === video, - ); - if (!configured) { - if (this.video === video) { - this.startStats(video); - } - return false; - } - senderParameters = configured.readback; - videoWarning = configured.warning; - appliedVideoProfile = effectiveVideoProfile; + if (videoChanged) await applyVideoCaptureProfile(video, previous); + await this.configure(previous, configureVideo); + this.videoWarning = say("host.fail.sfuParams"); + return false; } catch { - if ( - !this.owns(room, generation) || - requestedRevision !== this.profileRevision || - this.video !== video - ) { - if (this.video === video) { - this.startStats(video); - } - return false; - } - videoSucceeded = false; - const failureWarning = say("host.fail.sfuParams"); - try { - await applyVideoCaptureProfile( - video.rawTrack, - previousVideoProfile, - ); - const rolledBack = await configurePublishedVideo( - video, - previousVideoProfile, - () => - this.owns(room, generation) && - requestedRevision === this.profileRevision && - this.video === video, - ); - if (!rolledBack) { - if (this.video === video) { - this.startStats(video); - } - return false; - } - senderParameters = rolledBack.readback; - videoWarning = mergeQualityWarnings( - failureWarning, - rolledBack.warning, - ); - } catch (rollbackError) { - if ( - this.owns(room, generation) && - requestedRevision === this.profileRevision && - this.video === video - ) { - await this.failClosed(room, generation); - throw rollbackError; - } - if (this.video === video) { - this.startStats(video); - } - return false; - } + this.fail("sender-config"); + return false; } } - - if ( - !this.owns(room, generation) || - requestedRevision !== this.profileRevision || - this.video !== video || - this.audio !== audio - ) { - if (updateVideo && this.video === video) { - this.startStats(video); - } - return false; - } - - let audioSenderParameters = this.audioSenderParameters; - let audioParametersSender = this.audioParametersSender; - let audioWarning = this.audioQualityWarning; - let audioSucceeded = true; - if (updateAudio && audio) { - let audioSender: RTCRtpSender | null = null; - try { - audioSender = publishedAudioSender(audio); - const ownedAudioSender = audioSender; - const configured = await configurePublishedAudio( - audio, - ownedAudioSender, - profile, - () => - this.ownsAudioMutation( - room, - generation, - requestedRevision, - audio, - ownedAudioSender, - ), - ); - if (!configured) { - if (updateVideo && this.video === video) { - this.startStats(video); - } - return false; - } - audioSenderParameters = configured.readback; - audioParametersSender = configured.sender; - audioWarning = configured.warning; - } catch (error) { - if ( - !this.owns(room, generation) || - requestedRevision !== this.profileRevision || - this.audio !== audio || - (audioSender !== null && - audio.publication.audioTrack?.sender !== audioSender) - ) { - if (updateVideo && this.video === video) { - this.startStats(video); - } - return false; - } - audioSucceeded = false; - audioWarning = audioSenderFailureWarning(error); - } - } - - if ( - !this.owns(room, generation) || - requestedRevision !== this.profileRevision || - this.video !== video || - this.audio !== audio - ) { - if (updateVideo && this.video === video) { - this.startStats(video); - } - return false; - } - this.senderParameters = senderParameters; - this.audioSenderParameters = audioSenderParameters; - this.audioParametersSender = audioParametersSender; - this.appliedVideoProfile = appliedVideoProfile; - this.videoQualityWarning = videoWarning; - this.audioQualityWarning = audioWarning; - if (updateVideo) { - this.startStats(video); - } - return videoSucceeded && audioSucceeded; }); } getQualityWarning(): string | null { - return mergeQualityWarnings( - this.videoQualityWarning, - this.audioQualityWarning, + return ( + [this.videoWarning, this.audioWarning].filter(Boolean).join("; ") || null ); } - getFailureStage(): SfuPublisherFailureStage | null { return this.failureStage; } - getSenderParameters(): VideoSenderParameterReadback | null { return this.senderParameters; } - getAudioSenderParameters(): AudioSenderParameterReadback | null { return this.audioSenderParameters; } setPaused(paused: boolean): void { this.paused = paused; - this.applyPausedState( - this.video?.rawTrack ?? null, - this.audio?.rawTrack ?? null, - ); - this.applyPausedState(this.replacementVideoTrack, this.replacementAudioTrack); + for (const track of this.ownedTracks) { + track.enabled = !paused; + if (track.kind === "audio") track.contentHint = "music"; + } } - private applyPausedState( - videoTrack: MediaStreamTrack | null, - audioTrack: MediaStreamTrack | null, - ): void { - if (videoTrack) videoTrack.enabled = !this.paused; - if (audioTrack) audioTrack.enabled = !this.paused; + async disconnect(): Promise { + if (this.closed) return; + this.closed = true; + const peer = this.peer; + this.peer = null; + peer?.close(); + for (const track of this.ownedTracks) this.releaseTrack(track); + this.video = null; + this.audio = null; + this.videoSender = null; + this.audioSender = null; + if (this.statsTimer !== null) clearInterval(this.statsTimer); + this.statsTimer = null; + this.resetStats(); } - private reapplyAudioAfterReconnect(room: Room, generation: number): void { - void this.enqueue(async () => { - if (!this.owns(room, generation) || this.state !== "active") { - return false; - } - const sdk = this.sdk; - if (!sdk) { - return false; - } + private fail(stage: SfuPublisherFailureStage): void { + if (this.closed) return; + this.failureStage = stage; + void this.disconnect(); + this.events.onDisconnected?.(); + } - const previousVideo = this.video; - const previousAudio = this.audio; - const previousVideoSender = - previousVideo?.publication.videoTrack?.sender ?? null; - const previousAudioSender = - previousAudio?.publication.audioTrack?.sender ?? null; - const currentVideo = previousVideo - ? currentPublishedTrack( - room, - sdk.Track.Source.ScreenShare, - previousVideo, - ) - : null; - const videoSender = currentVideo?.publication.videoTrack?.sender ?? null; - if (!currentVideo || !videoSender) { - this.failureStage = "transport"; - await this.failClosed(room, generation); - return false; - } - const currentAudio = previousAudio - ? currentPublishedTrack( - room, - sdk.Track.Source.ScreenShareAudio, - previousAudio, - ) - : null; - const audioSender = currentAudio?.publication.audioTrack?.sender ?? null; - if (previousAudio && (!currentAudio || !audioSender)) { - this.failureStage = "transport"; - await this.failClosed(room, generation); - return false; - } - const videoChanged = - currentVideo.publication !== previousVideo?.publication || - videoSender !== previousVideoSender; - const audioChanged = - currentAudio?.publication !== previousAudio?.publication || - audioSender !== previousAudioSender; - const video = videoChanged ? currentVideo : previousVideo; - const audio = audioChanged ? currentAudio : previousAudio; - if (videoChanged) { - this.stopStats(); - const profile = this.desiredProfile; - if (!profile) { - this.failureStage = "transport"; - await this.failClosed(room, generation); - return false; - } - const previousRawTrack = video.rawTrack; - const nextRawTrack = cloneSenderVideoTrack(video.sourceTrack); - let retainedNextTrack = false; - try { - await applyVideoCaptureProfile(nextRawTrack, profile); - nextRawTrack.enabled = !this.paused; - await replacePublishedTrack(video, nextRawTrack); - if (!this.owns(room, generation)) { - return false; - } - nextRawTrack.enabled = !this.paused; - video.rawTrack = nextRawTrack; - retainedNextTrack = true; - previousRawTrack.stop(); - } catch { - this.failureStage = "transport"; - await this.failClosed(room, generation); - return false; - } finally { - if (!retainedNextTrack) { - nextRawTrack.stop(); - } - } - } - this.video = video; - this.audio = audio; - if (videoChanged) { - this.senderParameters = null; - this.videoQualityWarning = null; - this.startStats(video); - } + private ownTrack(track: T): T { + if (track) this.ownedTracks.add(track); + return track; + } - const profile = this.desiredProfile; - const profileRevision = this.profileRevision; - if (!audio || !profile) { - this.audioSenderParameters = null; - this.audioParametersSender = null; - this.audioQualityWarning = null; - return true; - } - retainPublishedAudioOptions(audio, profile, sdk); + private releaseTrack(track: MediaStreamTrack | null): void { + if (track && this.ownedTracks.delete(track)) track.stop(); + } - if (!audioSender) { - this.failureStage = "transport"; - await this.failClosed(room, generation); + private restartIce(): Promise { + return this.enqueue(async () => { + const peer = this.peer; + if (!peer || !this.profile || peer.pc.signalingState !== "stable") return false; - } - if ( - audio.publication !== previousAudio?.publication || - this.audioParametersSender !== audioSender - ) { - this.audioSenderParameters = null; - this.audioParametersSender = null; - this.audioQualityWarning = null; - } try { - const ownedAudioSender = audioSender; - const requestedMaxBitrate = screenAudioBitrate( - profile.screenAudioQuality, + await peer.sendDescription( + await peer.pc.createOffer({ iceRestart: true }), + this.media(this.profile), ); - const appliedMaxBitrate = - ownedAudioSender.getParameters().encodings[0]?.maxBitrate ?? null; - if (appliedMaxBitrate === requestedMaxBitrate) { - this.audioSenderParameters = { - requestedMaxBitrate, - appliedMaxBitrate, - mismatch: false, - }; - this.audioParametersSender = ownedAudioSender; - this.audioQualityWarning = null; - return true; - } - const configured = await configurePublishedAudio( - audio, - ownedAudioSender, - profile, - () => - this.ownsAudioMutation( - room, - generation, - profileRevision, - audio, - ownedAudioSender, - ), - ); - if (!configured) { - return false; - } - this.audioSenderParameters = configured.readback; - this.audioParametersSender = configured.sender; - this.audioQualityWarning = configured.warning; return true; - } catch (error) { - if ( - !this.owns(room, generation) || - profileRevision !== this.profileRevision || - this.audio !== audio || - (audioSender !== null && - audio.publication.audioTrack?.sender !== audioSender) - ) { - return false; - } - this.audioQualityWarning = audioSenderFailureWarning(error); + } catch { return false; } }); } - async disconnect(): Promise { - const room = this.invalidate(); - if (room) { - await safeDisconnect(room); - } - } - - private enqueue(operation: () => Promise): Promise { - const result = this.operationTail.then(operation, operation); - this.operationTail = result.then( + private enqueue(work: () => Promise): Promise { + const next = this.operationTail.then(work, work); + this.operationTail = next.then( () => undefined, () => undefined, ); - return result; + return next; } - private requireOwnedRoom( - generation: number, - expectedState: PublisherState | PublisherState[], - ): Room | null { - if (!this.ownsGeneration(generation)) { - return null; - } - const expected = Array.isArray(expectedState) - ? expectedState - : [expectedState]; - if (!expected.includes(this.state)) { - throw new Error(`SFU publisher is ${this.state}, expected ${expected.join(" or ")}`); - } - if (!this.room) { - throw new Error("SFU publisher room is unavailable"); - } - return this.room; - } - - private ownsGeneration(generation: number): boolean { - return this.generation === generation && this.state !== "disconnected"; - } - - private owns(room: Room, generation: number): boolean { - return this.room === room && this.ownsGeneration(generation); - } - - private ownsAudioMutation( - room: Room, - generation: number, - profileRevision: number, - audio: PublishedTrack, - sender: RTCRtpSender, - ): boolean { - return ( - this.owns(room, generation) && - this.state === "active" && - this.profileRevision === profileRevision && - this.audio === audio && - audio.publication.audioTrack?.sender === sender - ); - } - - private invalidate(): Room | null { - if (this.state === "disconnected") { - return null; - } - ++this.generation; - this.stopStats(); - this.state = "disconnected"; - const room = this.room; - releasePublishedTrack(this.video); - this.room = null; - this.sdk = null; - this.video = null; - this.audio = null; - this.desiredProfile = null; - this.appliedVideoProfile = null; - this.startupVideoProfilePending = false; - ++this.profileRevision; - this.senderParameters = null; - this.audioSenderParameters = null; - this.audioParametersSender = null; - this.videoQualityWarning = null; - this.audioQualityWarning = null; - return room; - } - - private async failClosed(room: Room, generation: number): Promise { - if (!this.owns(room, generation)) { - return; - } - this.invalidate(); - await safeDisconnect(room); - this.notifyTerminalDisconnect(); - } - - private notifyTerminalDisconnect(): void { - if (this.terminalNotified) { - return; - } - this.terminalNotified = true; - this.events.onDisconnected?.(); - } - - private retainSenderParameters( - configuration: PublishedVideoConfiguration, - audioSenderParameters: AudioSenderParameterReadback | null, - audioWarning: string | null, - audioParametersSender: RTCRtpSender | null, - ): void { - this.senderParameters = configuration.readback; - this.audioSenderParameters = audioSenderParameters; - this.audioParametersSender = audioParametersSender; - this.videoQualityWarning = configuration.warning; - this.audioQualityWarning = audioWarning; - } - - private startStats(video: PublishedTrack): void { - this.stopStats(); - this.statsVideo = video; - this.statsTimer = setInterval(() => { - void this.updateStats(video); - }, STATS_INTERVAL_MS); - } - - private async updateStats(video: PublishedTrack): Promise { - if ( - this.state !== "active" || - this.statsVideo !== video || - this.video !== video - ) { - return; - } - const videoSender = video.publication.videoTrack?.sender; - const audio = this.audio; - const audioSender = audio?.publication.audioTrack?.sender ?? null; - if ( - !videoSender || - videoSender.track !== video.rawTrack || - (audio !== null && - (!audioSender || audioSender.track !== audio.rawTrack)) - ) { - this.resetStatsIdentity(); - return; - } - const audioTrack = audio?.rawTrack ?? null; - let identity = this.statsIdentity; - if ( - !identity || - identity.video !== video || - identity.videoSender !== videoSender || - identity.videoTrack !== video.rawTrack || - identity.audio !== audio || - identity.audioSender !== audioSender || - identity.audioTrack !== audioTrack - ) { - this.resetStatsIdentity(); - identity = { - video, - videoSender, - videoTrack: video.rawTrack, - audio, - audioSender, - audioTrack, - accumulator: createStatsAccumulator(), - nativeQualityAccumulator: createNativeSenderQualityAccumulator(), - }; - this.statsIdentity = identity; - } - if (this.statsInFlight === identity) { - return; - } - this.statsInFlight = identity; - try { - const report = mergeStatsReports( - await Promise.all([ - identity.videoSender.getStats(), - identity.audioSender?.getStats(), - ]), - ); - if ( - !report || - this.state !== "active" || - this.statsIdentity !== identity || - this.video !== identity.video || - identity.video.rawTrack !== identity.videoTrack || - identity.video.publication.videoTrack?.sender !== - identity.videoSender || - identity.videoSender.track !== identity.videoTrack || - this.audio !== identity.audio || - (identity.audio?.rawTrack ?? null) !== identity.audioTrack || - (identity.audio?.publication.audioTrack?.sender ?? null) !== - identity.audioSender || - (identity.audioSender?.track ?? null) !== identity.audioTrack - ) { - return; - } - const nativeQuality = collectNativeSenderQualityFromReport( - report, - identity.videoTrack.id, - identity.nativeQualityAccumulator, - ); - const connectionMetrics = collectConnectionMetricsFromReport( - report, - "send", - identity.accumulator, - { - trackIdentifier: identity.videoTrack.id, - rid: highestActiveVideoRid(report, identity.videoTrack.id), - audioTrackIdentifier: identity.audioTrack?.id ?? null, - }, - ); - const videoEncodings = identity.videoSender.getParameters().encodings; - const metrics = { - ...connectionMetrics, - videoEncodingCount: - videoEncodings.length > 0 ? videoEncodings.length : null, - activeVideoEncodingCount: - videoEncodings.length > 0 - ? videoEncodings.filter((encoding) => encoding.active !== false) - .length - : null, - bitrateKbps: - nativeQuality.bitrateKbps ?? connectionMetrics.bitrateKbps, - nativeEdgeQualityState: nativeQuality.nativeEdgeQualityState, - ...(nativeQuality.nativeEdgeQualityState === "unknown" - ? {} - : { - qualityLimitationReason: - nativeQuality.qualityLimitationReason, - sampleTimestampMs: nativeQuality.sampleTimestampMs, - sampleWindowMs: nativeQuality.sampleWindowMs, - intervalFramesEncoded: nativeQuality.intervalFramesEncoded, - }), - ...captureMetrics(identity.videoTrack), - }; - if ( - this.startupVideoProfilePending && - maxEncodedVideoFrames(report, identity.videoTrack.id) >= - STARTUP_VIDEO_ENCODED_FRAMES - ) { - this.startupVideoProfilePending = false; - const desiredProfile = this.desiredProfile; - if (desiredProfile) { - void this.updateProfile(desiredProfile); - } - } - this.events.onStats?.(metrics); - } catch { - // Stats are observational and must never disrupt active SFU media. - } finally { - if (this.statsInFlight === identity) { - this.statsInFlight = null; - } - } - } - - private stopStats(): void { - if (this.statsTimer !== null) { - clearInterval(this.statsTimer); - this.statsTimer = null; - } - this.statsVideo = null; - this.statsInFlight = null; - this.resetStatsIdentity(); - } - - private resetStatsIdentity(): void { - const hadIdentity = this.statsIdentity !== null; - this.statsIdentity = null; - if (hadIdentity) { - this.events.onStats?.(null); - } - } -} - -async function publishTrack( - room: Room, - rawTrack: MediaStreamTrack, - source: Track.Source, - options: TrackPublishOptions, -): Promise { - const publication = await room.localParticipant.publishTrack(rawTrack, { - source, - ...options, - }); - if (!publication.track) { - throw new Error("SFU did not retain the published track"); - } - return { - publication, - sourceTrack: rawTrack, - rawTrack, - stopOnRelease: false, - }; -} - -async function publishVideoTrack( - room: Room, - sourceTrack: MediaStreamTrack, - profile: QualityProfile, - source: Track.Source, - options: TrackPublishOptions, -): Promise { - const rawTrack = cloneSenderVideoTrack(sourceTrack); - try { - await applyVideoCaptureProfile(rawTrack, profile); - const published = await publishTrack(room, rawTrack, source, options); - published.sourceTrack = sourceTrack; - published.stopOnRelease = true; - return published; - } catch (error) { - rawTrack.stop(); - throw error; - } -} - -async function replacePublishedTrack( - published: PublishedTrack, - nextTrack: MediaStreamTrack, -): Promise { - const localTrack = published.publication.track; - if (!localTrack) { - throw new Error("SFU publication has no local track"); - } - await localTrack.replaceTrack(nextTrack); -} - -async function unpublishTrack(room: Room, published: PublishedTrack): Promise { - try { - const localTrack = published.publication.track; - if (!localTrack) { - throw new Error("SFU publication has no local track"); - } - await room.localParticipant.unpublishTrack(localTrack, false); - } finally { - releasePublishedTrack(published); - } -} - -function releasePublishedTrack(published: PublishedTrack | null): void { - if (!published?.stopOnRelease) { - return; - } - published.stopOnRelease = false; - published.rawTrack.stop(); -} - -async function configurePublishedVideo( - published: PublishedTrack, - profile: QualityProfile, - ownsPublication: () => boolean, -): Promise { - const videoTrack = published.publication.videoTrack; - const sender = videoTrack?.sender; - if (!sender) { - throw new Error("SFU video publication has no RTP sender"); - } - const previousPreference = videoTrack.publishOptions?.degradationPreference; - await videoTrack.setDegradationPreference(profile.degradationPreference); - if (!ownsPublication()) { - if (previousPreference) { - await videoTrack.setDegradationPreference(previousPreference); - } - return null; - } - const readback = await configureVideoSender(sender, profile); - if (!ownsPublication()) { - if (previousPreference) { - await videoTrack.setDegradationPreference(previousPreference); - } - return null; - } - const retainedPublishOptions = { - ...published.publication.options, - ...videoTrack.publishOptions, - ...videoPublishOptions(profile, publishedVideoCodec(published)), - }; - published.publication.options = retainedPublishOptions; - videoTrack.publishOptions = retainedPublishOptions; - return { - readback, - warning: senderParameterWarning(readback), - }; -} - -async function configurePublishedAudio( - published: PublishedTrack, - sender: RTCRtpSender, - profile: QualityProfile, - ownsPublication: () => boolean, -): Promise { - if (published.publication.audioTrack?.sender !== sender) { - return null; - } - const readback = await configureScreenAudioSender( - sender, - profile.screenAudioQuality, - ); - if ( - !ownsPublication() || - published.publication.audioTrack?.sender !== sender - ) { - return null; - } - return { - sender, - readback, - warning: audioSenderParameterWarning(readback), - }; -} - -function publishedAudioSender(published: PublishedTrack): RTCRtpSender { - const sender = published.publication.audioTrack?.sender; - if (!sender) { - throw new Error("SFU audio publication has no RTP sender"); - } - return sender; -} - -function currentPublishedTrack( - room: Room, - source: Track.Source, - expected: PublishedTrack, -): PublishedTrack | null { - const publication = room.localParticipant.getTrackPublication(source); - const rawTrack = publication?.track?.mediaStreamTrack; - if ( - !publication || - !rawTrack || - rawTrack !== expected.rawTrack - ) { - return null; - } - return { - publication, - sourceTrack: expected.sourceTrack, - rawTrack, - stopOnRelease: expected.stopOnRelease, - }; -} - -function retainPublishedAudioOptions( - published: PublishedTrack, - profile: QualityProfile, - sdk: LiveKit, -): void { - published.publication.options = { - ...published.publication.options, - ...audioPublishOptions(sdk, profile.screenAudioQuality), - }; -} - -function audioSenderFailureWarning(_error: unknown): string { - return say("host.fail.sfuAudioParams"); -} - -function mergeQualityWarnings(...warnings: Array): string | null { - const present = warnings.filter( - (warning): warning is string => warning !== null, - ); - return present.length > 0 ? present.join("；") : null; -} - -function publishedVideoCodec(published: PublishedTrack): BrowserVideoCodec { - return published.publication.options?.videoCodec === "h264" - ? "h264" - : "vp8"; -} - -function videoPublishOptions( - profile: QualityProfile, - videoCodec: BrowserVideoCodec, -): TrackPublishOptions { - return { - backupCodec: false, - videoCodec, - screenShareEncoding: { + private encodings(profile: QualityProfile): RTCRtpEncodingParameters[] { + const settings = this.video?.getSettings(); + const ceiling = QUALITY_RESOLUTIONS[profile.resolution]; + const width = settings?.width ?? ceiling.width; + const height = settings?.height ?? ceiling.height; + const scale = Math.max(1, width / ceiling.width, height / ceiling.height); + // LiveKit 2.22.1 publishUtils: screen share uses half size, same FPS, + // and quarter bitrate with a 150 kbps floor. WebRTC budgets the entire PC. + const high = { + rid: this.layerCount === 2 ? "h" : "q", + active: this.activeCount >= this.layerCount, + scaleResolutionDownBy: scale, maxBitrate: profile.maxBitrate, maxFramerate: profile.maxFramerate, - }, - degradationPreference: profile.degradationPreference, - }; -} - -function audioPublishOptions( - sdk: LiveKit, - quality: ScreenAudioQuality | undefined, -): TrackPublishOptions { - const resolvedQuality = resolveScreenAudioQuality(quality); - const audioPreset = - resolvedQuality === "saver" - ? sdk.AudioPresets.musicStereo - : resolvedQuality === "music" - ? sdk.AudioPresets.musicHighQualityStereo - : { maxBitrate: screenAudioBitrate(resolvedQuality) }; - return { - audioPreset, - forceStereo: true, - dtx: false, - red: false, - }; -} - -async function safeDisconnect(room: Room): Promise { - const existing = roomDisconnects.get(room); - if (existing) { - await existing; - return; + }; + return this.layerCount === 2 + ? [ + { + rid: "q", + active: this.activeCount > 0, + scaleResolutionDownBy: 2 * scale, + maxBitrate: Math.max(150_000, Math.floor(profile.maxBitrate / 4)), + maxFramerate: profile.maxFramerate, + }, + high, + ] + : [high]; + } + + private media(profile: QualityProfile): SfuMedia { + const settings = this.video!.getSettings(); + const ceiling = QUALITY_RESOLUTIONS[profile.resolution]; + return { + codec: this.codec, + layers: this.encodings(profile).map((encoding) => ({ + rid: encoding.rid!, + width: Math.max( + 1, + Math.floor( + (settings.width ?? ceiling.width) / encoding.scaleResolutionDownBy!, + ), + ), + height: Math.max( + 1, + Math.floor( + (settings.height ?? ceiling.height) / + encoding.scaleResolutionDownBy!, + ), + ), + bitrate: encoding.maxBitrate!, + })), + audio: this.audio !== null, + audioBitrate: screenAudioBitrate(profile.screenAudioQuality), + }; + } + + private async configure( + profile: QualityProfile, + configureVideo = true, + ): Promise { + const peer = this.peer; + const sender = this.videoSender; + if (!peer || !sender) return false; + if (configureVideo) { + const parameters = sender.getParameters(); + const encodings = this.encodings(profile); + parameters.encodings.forEach((encoding, index) => + Object.assign(encoding, encodings[index]), + ); + await sender.setParameters(parameters); + if (this.peer !== peer) return false; + const readback = await configureVideoSender( + sender, + this.startupPending ? startupVideoProfile(profile) : profile, + ); + if (this.peer !== peer) return false; + this.senderParameters = readback; + this.videoWarning = senderParameterWarning(readback); + } + if (!this.audio || !this.audioSender) { + this.audioSenderParameters = null; + this.audioWarning = null; + return true; + } + try { + const audio = await configureScreenAudioSender( + this.audioSender, + profile.screenAudioQuality, + ); + if (this.peer !== peer) return false; + this.audioSenderParameters = audio; + this.audioWarning = audioSenderParameterWarning(audio); + return true; + } catch { + this.audioWarning = say("host.fail.sfuAudioParams"); + return false; + } + } + + private startStats(): void { + this.resetStats(); + this.statsTimer = setInterval(() => void this.updateStats(), 2_000); + } + + private resetStats(): void { + this.stats = createStatsAccumulator(); + this.nativeStats = createNativeSenderQualityAccumulator(); + this.statsInFlight = null; + this.events.onStats?.(null); + } + + private async updateStats(): Promise { + const peer = this.peer; + const video = this.video; + const sender = this.videoSender; + const stats = this.stats; + const nativeStats = this.nativeStats; + if (!peer || !video || !sender || this.statsInFlight === stats) return; + this.statsInFlight = stats; + try { + const report = await peer.pc.getStats(); + if (this.peer !== peer || this.video !== video || this.stats !== stats) + return; + const native = collectNativeSenderQualityFromReport( + report, + video.id, + nativeStats, + ); + const connection = collectConnectionMetricsFromReport( + report, + "send", + stats, + { + trackIdentifier: video.id, + rid: highestActiveVideoRid(report, video.id), + audioTrackIdentifier: this.audio?.id ?? null, + }, + ); + const encodings = sender.getParameters().encodings; + this.events.onStats?.({ + ...connection, + ...captureMetrics(video), + videoEncodingCount: encodings.length, + activeVideoEncodingCount: encodings.filter( + (encoding) => encoding.active !== false, + ).length, + bitrateKbps: native.bitrateKbps ?? connection.bitrateKbps, + nativeEdgeQualityState: native.nativeEdgeQualityState, + ...(native.nativeEdgeQualityState === "unknown" + ? {} + : { + qualityLimitationReason: native.qualityLimitationReason, + sampleTimestampMs: native.sampleTimestampMs, + sampleWindowMs: native.sampleWindowMs, + intervalFramesEncoded: native.intervalFramesEncoded, + }), + }); + if ( + this.startupPending && + maxEncodedVideoFrames(report, video.id) >= STARTUP_VIDEO_ENCODED_FRAMES + ) { + this.startupPending = false; + if (this.profile) void this.updateProfile(this.profile); + } + } catch { + /* Diagnostics do not own media lifetime. */ + } finally { + if (this.statsInFlight === stats) this.statsInFlight = null; + } } - const disconnecting = room.disconnect(false).catch(() => undefined); - roomDisconnects.set(room, disconnecting); - await disconnecting; } -function requiredVideoTrack(stream: MediaStream): MediaStreamTrack { +function requiredVideo(stream: MediaStream): MediaStreamTrack { const track = stream.getVideoTracks()[0]; - if (!track) { - throw new Error(say("host.capture.noSource")); - } + if (!track || track.readyState === "ended") + throw new Error("SFU source has no live video track"); return track; } diff --git a/src/client/sfu/subscriber.ts b/src/client/sfu/subscriber.ts index dc6a654..7a868d3 100644 --- a/src/client/sfu/subscriber.ts +++ b/src/client/sfu/subscriber.ts @@ -1,24 +1,16 @@ -import type { - RemoteParticipant, - RemoteTrack, - RemoteTrackPublication, - Room, - Track, -} from "livekit-client"; - -import type { SfuConnectionConfig } from "./publisher"; +import type { SfuSignalMessage } from "../../shared/protocol"; import type { ConnectionMetrics } from "../types"; import { collectConnectionMetricsFromReport, createStatsAccumulator, decodedVideoFrames, - mergeStatsReports, - type StatsAccumulator, } from "../webrtc/stats"; +import { preferScreenAudioStereo } from "../webrtc/screen-audio-sdp"; import { observeDecodedFrameProof } from "../media/decoded-frame-proof"; -import { sfuRoomConnectOptions } from "./connection-options"; +import { SfuPeer, type SfuConnectionConfig } from "./peer"; interface SubscriberEvents { + send: (message: SfuSignalMessage) => boolean; onStream: (stream: MediaStream | null) => void; onVideoAvailability?: (available: boolean) => void; onStats?: (metrics: ConnectionMetrics) => void; @@ -29,693 +21,233 @@ interface SubscriberEvents { } interface SubscribedTrack { - sid: string; - track: RemoteTrack; - mediaStreamTrack: MediaStreamTrack; + track: MediaStreamTrack; + receiver: RTCRtpReceiver; onEnded: () => void; } -interface AudioStatsSample { - track: SubscribedTrack; - report: RTCStatsReport | null; -} - -interface KnownHostPublication { - publication: RemoteTrackPublication; - participant: RemoteParticipant; -} - -type SubscriberState = - | "idle" - | "connecting" - | "prepared" - | "active" - | "disconnected"; -type LiveKit = typeof import("livekit-client"); - -const HOST_IDENTITY = "host"; -const STATS_INTERVAL_MS = 2_000; -const roomDisconnects = new WeakMap>(); - export class SfuSubscriber { - private room: Room | null = null; - private sdk: LiveKit | null = null; + private peer: SfuPeer | null = null; private video: SubscribedTrack | null = null; private audio: SubscribedTrack | null = null; private readonly stream = new MediaStream(); + private active = false; + private closed = false; + private restartPending = false; private streamEmitted = false; - private readonly desiredTrackSids = new Set(); - private readonly knownHostPublications = new Map< - string, - KnownHostPublication - >(); - private state: SubscriberState = "idle"; - private statsAccumulator: StatsAccumulator = createStatsAccumulator(); + private stats = createStatsAccumulator(); + private statsInFlight: typeof this.stats | null = null; private statsTimer: ReturnType | null = null; - private statsInFlight: StatsAccumulator | null = null; - private audioStatsInFlight: SubscribedTrack | null = null; - private decodedFrameProofMode: "fresh" | "progress" | null = null; - private stopDecodedFrameObserver: (() => void) | null = null; - private generation = 0; - private terminalNotified = false; + private proofMode: "fresh" | "progress" | null = null; + private stopProof: (() => void) | null = null; constructor(private readonly events: SubscriberEvents) {} async connect(config: SfuConnectionConfig): Promise { - if (this.state !== "idle") { + if (this.peer || this.closed) throw new Error("SFU subscriber cannot be connected twice"); - } + this.peer = new SfuPeer(config, { + send: this.events.send, + onTrack: (event) => this.addTrack(event), + onState: (state) => { + if (state === "failed") this.fail(); + else if (state === "connected") { + this.restartPending = false; + this.events.onState?.("connected"); + this.startProof(); + } else if (state === "disconnected") this.reconnect(); + }, + }); + return true; + } - const generation = ++this.generation; - this.state = "connecting"; + updateConfig(config: SfuConnectionConfig): void { + this.peer?.updateConfig(config); + } + async acceptSignal(message: SfuSignalMessage): Promise { + const peer = this.peer; + if (!peer) return; try { - const sdk = await import("livekit-client"); - if (!this.ownsGeneration(generation)) { - return false; - } - - const room = new sdk.Room({ disconnectOnPageLeave: false }); - this.room = room; - this.sdk = sdk; - this.bindRoomEvents(room, sdk, generation); - - await room.connect(config.url, config.token, sfuRoomConnectOptions()); - if (!this.owns(room, generation)) { - await safeDisconnect(room); - return false; - } - - this.state = "prepared"; - return true; - } catch (error) { - if (!this.ownsGeneration(generation)) { - return false; - } - const room = this.invalidate(false); - if (room) { - await safeDisconnect(room); - } - throw error; + await peer.acceptSignal(message, async (description) => { + if (description.type !== "offer") + throw new Error("SFU subscriber expected an offer"); + await peer.pc.setRemoteDescription(description); + if (this.peer !== peer) return; + await peer.sendDescription( + preferScreenAudioStereo(await peer.pc.createAnswer()), + ); + this.restartPending = false; + }); + } catch { + if (this.peer === peer) this.fail(); } } activate(): boolean { - if (this.state === "active") { - return true; - } - if (this.state !== "prepared" || !this.room || !this.sdk) { - throw new Error(`SFU subscriber is ${this.state}, expected prepared`); - } - - const room = this.room; - const generation = this.generation; - this.state = "active"; - try { - this.startStats(); - this.reconcileHostSubscriptions(room, this.sdk, generation); - return true; - } catch (error) { - if (this.owns(room, generation)) { - try { - this.rollbackSubscriptions(room); - this.state = "prepared"; - this.clearMedia(true); - } catch { - void this.failClosed(room, generation); - } - } - throw error; - } + if (!this.peer || this.closed) return false; + if (this.active) return true; + this.active = true; + this.statsTimer = setInterval(() => void this.updateStats(), 2_000); + this.emitStream(); + this.startProof(); + return true; } deactivate(): boolean { - if (this.state === "prepared") { - return true; - } - if (this.state !== "active" || !this.room) { - if (this.state === "disconnected") { - return false; - } - throw new Error(`SFU subscriber is ${this.state}, expected active`); - } + void this.disconnect(); + return true; + } - const room = this.room; - try { - this.rollbackSubscriptions(room); - } catch (error) { - void this.failClosed(room, this.generation); - throw error; - } - this.state = "prepared"; - this.clearMedia(true); + reconnect(): boolean { + if ( + !this.active || + this.restartPending || + !this.peer?.send({ kind: "subscribe" }) + ) + return false; + this.restartPending = true; + this.events.onState?.("reconnecting"); return true; } armDecodedFrameProof(requireProgress = false): void { - this.cancelDecodedFrameObserver(); - this.decodedFrameProofMode = requireProgress ? "progress" : "fresh"; - this.startDecodedFrameProof(); + this.proofMode = requireProgress ? "progress" : "fresh"; + this.startProof(); } stopDecodedFrameProof(): void { - this.decodedFrameProofMode = null; - this.cancelDecodedFrameObserver(); + this.proofMode = null; + this.stopProof?.(); + this.stopProof = null; } async disconnect(): Promise { - const room = this.invalidate(true); - if (room) { - await safeDisconnect(room); - } - } - - private bindRoomEvents( - room: Room, - sdk: LiveKit, - generation: number, - ): void { - room.on( - sdk.RoomEvent.TrackPublished, - (publication: RemoteTrackPublication, participant: RemoteParticipant) => { - if (!this.owns(room, generation)) { - return; - } - try { - this.rememberHostPublication(publication, participant, sdk); - } catch { - void this.failClosed(room, generation); - } - }, - ); - room.on( - sdk.RoomEvent.ParticipantConnected, - (participant: RemoteParticipant) => { - if (!this.owns(room, generation)) { - return; - } - try { - this.rememberHostParticipant(participant, sdk); - } catch { - void this.failClosed(room, generation); - } - }, - ); - room.on( - sdk.RoomEvent.TrackSubscribed, - ( - track: RemoteTrack, - publication: RemoteTrackPublication, - participant: RemoteParticipant, - ) => { - if ( - this.state !== "active" || - !this.owns(room, generation) || - participant.identity !== HOST_IDENTITY || - !this.desiredTrackSids.has(publication.trackSid) - ) { - return; - } - this.addTrack( - room, - generation, - publication.trackSid, - publication.source, - track, - sdk.Track, - ); - }, - ); - room.on( - sdk.RoomEvent.TrackUnsubscribed, - ( - track: RemoteTrack, - publication: RemoteTrackPublication, - participant: RemoteParticipant, - ) => { - if (this.owns(room, generation) && participant.identity === HOST_IDENTITY) { - this.removeTrack(publication.trackSid, track.mediaStreamTrack); - } - }, - ); - room.on( - sdk.RoomEvent.TrackUnpublished, - (publication: RemoteTrackPublication, participant: RemoteParticipant) => { - if (this.owns(room, generation) && participant.identity === HOST_IDENTITY) { - this.knownHostPublications.delete(publication.trackSid); - this.desiredTrackSids.delete(publication.trackSid); - this.removeTrack(publication.trackSid); - } - }, - ); - room.on( - sdk.RoomEvent.ParticipantDisconnected, - (participant: RemoteParticipant) => { - if (this.owns(room, generation) && participant.identity === HOST_IDENTITY) { - for (const [sid, known] of this.knownHostPublications) { - if (known.participant === participant) { - this.knownHostPublications.delete(sid); - } - } - this.desiredTrackSids.clear(); - // LiveKit full reconnect removes participants before it emits - // Reconnecting. Keep any route-owned proof intent until the route is - // deactivated or the subscriber terminates. - this.clearMedia(true, true); - } - }, - ); - room.on(sdk.RoomEvent.Reconnecting, () => { - if (this.owns(room, generation) && this.state === "active") { - this.events.onState?.("reconnecting"); - } - }); - room.on(sdk.RoomEvent.Reconnected, () => { - if (this.owns(room, generation) && this.state === "active") { - this.reconcileHostSubscriptions(room, sdk, generation); - this.startDecodedFrameProof(); - this.events.onState?.("connected"); - void this.updateStats(); - } - }); - room.on(sdk.RoomEvent.Disconnected, () => { - if (this.owns(room, generation)) { - const notify = this.state !== "connecting"; - this.invalidate(true); - if (notify) { - this.notifyTerminalDisconnect(); - } - } - }); - } - - private subscribeIfAllowed( - publication: RemoteTrackPublication, - participant: RemoteParticipant, - sdk: LiveKit, - ): void { - if ( - participant.identity !== HOST_IDENTITY || - !isScreenSource(publication.source, sdk.Track) - ) { - return; - } - if (this.desiredTrackSids.has(publication.trackSid)) { - return; - } - this.desiredTrackSids.add(publication.trackSid); - publication.setSubscribed(true); - } - - private rememberHostParticipant( - participant: RemoteParticipant, - sdk: LiveKit, - ): void { - if (participant.identity !== HOST_IDENTITY) { - return; - } - for (const publication of participant.trackPublications.values()) { - this.rememberHostPublication(publication, participant, sdk); - } - } - - private rememberHostPublication( - publication: RemoteTrackPublication, - participant: RemoteParticipant, - sdk: LiveKit, - ): void { - if ( - participant.identity !== HOST_IDENTITY || - !isScreenSource(publication.source, sdk.Track) - ) { - return; - } - this.knownHostPublications.set(publication.trackSid, { - publication, - participant, - }); - if (this.state === "active") { - this.subscribeIfAllowed(publication, participant, sdk); - } - } - - private reconcileHostSubscriptions( - room: Room, - sdk: LiveKit, - generation: number, - ): void { - if (!this.owns(room, generation) || this.state !== "active") { - return; - } - const host = room.remoteParticipants.get(HOST_IDENTITY); - if (host) { - this.rememberHostParticipant(host, sdk); - } - for (const { publication, participant } of this.knownHostPublications.values()) { - this.subscribeIfAllowed(publication, participant, sdk); - } - } - - private rollbackSubscriptions(room: Room): void { - const host = room.remoteParticipants.get(HOST_IDENTITY); - for (const publication of host?.trackPublications.values() ?? []) { - if (this.desiredTrackSids.has(publication.trackSid)) { - publication.setSubscribed(false); - } - } - this.desiredTrackSids.clear(); - } - - private addTrack( - room: Room, - generation: number, - sid: string, - source: Track.Source, - track: RemoteTrack, - trackType: typeof Track, - ): void { - const isVideo = source === trackType.Source.ScreenShare; - let previous: SubscribedTrack | null; - if (isVideo) { - previous = this.video; - } else if (source === trackType.Source.ScreenShareAudio) { - previous = this.audio; - } else { - return; - } - const mediaStreamTrack = track.mediaStreamTrack; - if (previous?.sid === sid && previous.mediaStreamTrack === mediaStreamTrack) { - return; - } + if (this.closed) return; + this.closed = true; + this.active = false; + const peer = this.peer; + this.peer = null; + peer?.close(); + this.stopDecodedFrameProof(); + if (this.statsTimer !== null) clearInterval(this.statsTimer); + this.statsTimer = null; const hadVideo = this.video !== null; - if (isVideo) { - this.cancelDecodedFrameObserver(); - } - if (previous) this.detachTrack(previous); - const subscribed = { - sid, + this.detach(this.video); + this.detach(this.audio); + this.video = null; + this.audio = null; + this.stats = createStatsAccumulator(); + if (hadVideo) this.events.onVideoAvailability?.(false); + if (this.streamEmitted) this.events.onStream(null); + this.streamEmitted = false; + } + + private fail(): void { + if (this.closed) return; + void this.disconnect(); + this.events.onDisconnected?.(); + } + + private addTrack({ track, receiver }: RTCTrackEvent): void { + if (track.kind !== "video" && track.kind !== "audio") return; + const isVideo = track.kind === "video"; + const previous = isVideo ? this.video : this.audio; + if (previous?.track === track) return; + this.detach(previous); + const subscribed: SubscribedTrack = { track, - mediaStreamTrack, + receiver, onEnded: () => { - if (this.state === "active" && this.owns(room, generation)) { - this.removeTrack(sid, mediaStreamTrack); - } + if ((isVideo ? this.video : this.audio) !== subscribed) return; + this.detach(subscribed); + if (isVideo) { + this.video = null; + this.stopProof?.(); + this.stopProof = null; + this.events.onVideoAvailability?.(false); + } else this.audio = null; + this.stats = createStatsAccumulator(); + this.emitStream(); }, }; - mediaStreamTrack.addEventListener("ended", subscribed.onEnded, { once: true }); - this.stream.addTrack(mediaStreamTrack); - if (isVideo) { - this.video = subscribed; - this.resetStatsAccumulator(); - } else { - this.audio = subscribed; - this.resetAudioStats(); - } + track.addEventListener("ended", subscribed.onEnded, { once: true }); + this.stream.addTrack(track); + if (isVideo) this.video = subscribed; + else this.audio = subscribed; + this.stats = createStatsAccumulator(); this.emitStream(); - if (!hadVideo && this.video) { - this.events.onVideoAvailability?.(true); - } - if (this.video) { - this.startStats(); - } if (isVideo) { - this.startDecodedFrameProof(); + this.events.onVideoAvailability?.(true); + this.startProof(); } } - private removeTrack(sid: string, track?: MediaStreamTrack): void { - const previousVideo = this.video; - const previousAudio = this.audio; - if (this.video?.sid === sid && (!track || this.video.mediaStreamTrack === track)) { - this.cancelDecodedFrameObserver(); - this.detachTrack(this.video); - this.video = null; - } - if (this.audio?.sid === sid && (!track || this.audio.mediaStreamTrack === track)) { - this.detachTrack(this.audio); - this.audio = null; - this.resetAudioStats(); - } - if (this.video === previousVideo && this.audio === previousAudio) { - return; - } - this.emitStream(); - if (!this.video && previousVideo) { - this.resetStatsAccumulator(); - this.events.onVideoAvailability?.(false); - } + private detach(subscribed: SubscribedTrack | null): void { + if (!subscribed) return; + subscribed.track.removeEventListener("ended", subscribed.onEnded); + this.stream.removeTrack(subscribed.track); } private emitStream(): void { - if (!this.video) { - return; - } + if (!this.active || !this.video) return; this.streamEmitted = true; this.events.onStream(this.stream); } - private detachTrack(track: SubscribedTrack): void { - track.mediaStreamTrack.removeEventListener("ended", track.onEnded); - this.stream.removeTrack(track.mediaStreamTrack); - } - - private startStats(): void { - if (this.statsTimer === null) { - this.statsTimer = setInterval(() => { - void this.updateStats(); - }, STATS_INTERVAL_MS); - } - void this.updateStats(); - } - - private startDecodedFrameProof(): void { - const room = this.room; + private startProof(): void { + this.stopProof?.(); + this.stopProof = null; + const peer = this.peer; const video = this.video; - const generation = this.generation; - const mode = this.decodedFrameProofMode; - if (!room || !video || !mode || this.state !== "active") { - return; - } - this.cancelDecodedFrameObserver(); - this.stopDecodedFrameObserver = observeDecodedFrameProof({ - readFramesDecoded: async () => { - const report = mergeStatsReports([ - await video.track.getRTCStatsReport(), - ]); - return report ? decodedVideoFrames(report) : null; - }, + const mode = this.proofMode; + if (!peer || !video || !mode || !this.active) return; + this.stopProof = observeDecodedFrameProof({ + readFramesDecoded: async () => + decodedVideoFrames(await video.receiver.getStats()), owns: () => - this.owns(room, generation) && - this.state === "active" && + this.peer === peer && + this.active && this.video === video && - this.decodedFrameProofMode === mode, + this.proofMode === mode, requireProgress: mode === "progress", onProof: () => { const accepted = this.events.onFirstDecodedFrame?.() ?? true; if (accepted) { - this.decodedFrameProofMode = null; - this.stopDecodedFrameObserver = null; + this.proofMode = null; + this.stopProof = null; } return accepted; }, }); } - private cancelDecodedFrameObserver(): void { - this.stopDecodedFrameObserver?.(); - this.stopDecodedFrameObserver = null; - } - private async updateStats(): Promise { - const room = this.room; + const peer = this.peer; const video = this.video; - const generation = this.generation; - const accumulator = this.statsAccumulator; - if (!room || this.state !== "active") { - return; - } - if (this.statsInFlight === accumulator || !video) { + const stats = this.stats; + if (!peer || !this.active) return; + if (!video || this.statsInFlight === stats) { this.events.onDecodedFrameSample?.(null); return; } - const ownsSample = (): boolean => - this.owns(room, generation) && - this.state === "active" && - this.video === video && - this.statsAccumulator === accumulator; - const audioStatsSample: AudioStatsSample | null = this.audio - ? { track: this.audio, report: null } - : null; - if (audioStatsSample) { - void this.updateAudioStats(room, generation, audioStatsSample); - } - this.statsInFlight = accumulator; + this.statsInFlight = stats; try { - const videoReport = await video.track.getRTCStatsReport(); - if (!ownsSample()) { + const report = await peer.pc.getStats(); + if (this.peer !== peer || this.video !== video || this.stats !== stats) return; - } - const report = mergeStatsReports([ - videoReport, - audioStatsSample?.track === this.audio - ? (audioStatsSample.report ?? undefined) - : undefined, - ]); - if (!report) { - this.events.onDecodedFrameSample?.(null); - return; - } const metrics = collectConnectionMetricsFromReport( report, "receive", - accumulator, + stats, ); this.events.onDecodedFrameSample?.(metrics.intervalFramesDecoded); this.events.onStats?.(metrics); } catch { - if (ownsSample()) { + if (this.peer === peer && this.stats === stats) this.events.onDecodedFrameSample?.(null); - } } finally { - if (this.statsInFlight === accumulator) { - this.statsInFlight = null; - } + if (this.statsInFlight === stats) this.statsInFlight = null; } } - - private async updateAudioStats( - room: Room, - generation: number, - sample: AudioStatsSample, - ): Promise { - const audio = sample.track; - if (this.audioStatsInFlight === audio) { - return; - } - this.audioStatsInFlight = audio; - try { - const report = await audio.track.getRTCStatsReport(); - if ( - report && - this.owns(room, generation) && - this.state === "active" && - this.audio === audio - ) { - sample.report = report; - } - } catch { - // Audio diagnostics must not affect video liveness. - } finally { - if (this.audioStatsInFlight === audio) { - this.audioStatsInFlight = null; - } - } - } - - private stopStats(): void { - if (this.statsTimer !== null) { - clearInterval(this.statsTimer); - this.statsTimer = null; - } - this.resetStatsAccumulator(); - } - - private resetStatsAccumulator(): void { - this.statsInFlight = null; - this.statsAccumulator = createStatsAccumulator(); - } - - private resetAudioStats(): void { - this.audioStatsInFlight = null; - } - - private clearMedia(notify: boolean, preserveDecodedFrameProof = false): void { - const hadStream = this.streamEmitted; - const hadVideo = this.video !== null; - if (preserveDecodedFrameProof) { - this.cancelDecodedFrameObserver(); - } else { - this.stopDecodedFrameProof(); - } - if (this.state !== "active") { - this.stopStats(); - } - if (this.video) { - this.detachTrack(this.video); - } - if (this.audio) { - this.detachTrack(this.audio); - } - this.video = null; - this.audio = null; - this.resetAudioStats(); - if (this.state === "active") { - this.resetStatsAccumulator(); - } - this.streamEmitted = false; - if (hadVideo) { - this.events.onVideoAvailability?.(false); - } - if (notify && hadStream) { - this.events.onStream(null); - } - } - - private ownsGeneration(generation: number): boolean { - return this.generation === generation && this.state !== "disconnected"; - } - - private owns(room: Room, generation: number): boolean { - return this.room === room && this.ownsGeneration(generation); - } - - private invalidate(notifyStream: boolean): Room | null { - if (this.state === "disconnected") { - return null; - } - ++this.generation; - this.state = "disconnected"; - const room = this.room; - this.room = null; - this.sdk = null; - this.desiredTrackSids.clear(); - this.knownHostPublications.clear(); - this.clearMedia(notifyStream); - return room; - } - - private async failClosed(room: Room, generation: number): Promise { - if (!this.owns(room, generation)) { - return; - } - this.invalidate(true); - await safeDisconnect(room); - this.notifyTerminalDisconnect(); - } - - private notifyTerminalDisconnect(): void { - if (this.terminalNotified) { - return; - } - this.terminalNotified = true; - this.events.onDisconnected?.(); - } -} - -function isScreenSource(source: Track.Source, trackType: typeof Track): boolean { - return ( - source === trackType.Source.ScreenShare || - source === trackType.Source.ScreenShareAudio - ); -} - -async function safeDisconnect(room: Room): Promise { - const existing = roomDisconnects.get(room); - if (existing) { - await existing; - return; - } - const disconnecting = room.disconnect(false).catch(() => undefined); - roomDisconnects.set(room, disconnecting); - await disconnecting; } diff --git a/src/client/types.ts b/src/client/types.ts index 25b731c..c607ae8 100644 --- a/src/client/types.ts +++ b/src/client/types.ts @@ -1,18 +1,12 @@ import type { NatTraversalPath } from "../shared/nat-candidate"; export type SignalConnectionState = - | "connecting" - | "connected" - | "reconnecting" - | "offline"; + "connecting" | "connected" | "reconnecting" | "offline"; export type MediaPath = "direct" | "unknown"; export type NativeEdgeQualityState = "unknown" | "healthy" | "degraded"; export type QualityWarningKind = - | "bandwidth" - | "cpu" - | "other" - | "configuration"; + "bandwidth" | "cpu" | "other" | "configuration"; export interface ConnectionMetrics { sampleTimestampMs: number | null; @@ -79,6 +73,7 @@ export interface ConnectionMetrics { encoderImplementation: string | null; powerEfficientEncoder: boolean | null; intervalFramesEncoded: number | null; + intervalFramesSent: number | null; intervalEncodeTimeMs: number | null; intervalEncodeMs: number | null; intervalDecodeMs: number | null; @@ -93,10 +88,10 @@ export interface PeerSnapshot { iceConnectionState: RTCIceConnectionState; metrics: ConnectionMetrics; error: string | null; - senderParameters?: import("./media/quality").VideoSenderParameterReadback | null; + senderParameters?: + import("./media/quality").VideoSenderParameterReadback | null; audioSenderParameters?: - | import("./media/quality").AudioSenderParameterReadback - | null; + import("./media/quality").AudioSenderParameterReadback | null; qualityWarning?: string | null; qualityWarningKind?: QualityWarningKind | null; } @@ -166,6 +161,7 @@ export const EMPTY_METRICS: ConnectionMetrics = { encoderImplementation: null, powerEfficientEncoder: null, intervalFramesEncoded: null, + intervalFramesSent: null, intervalEncodeTimeMs: null, intervalEncodeMs: null, intervalDecodeMs: null, diff --git a/src/client/webrtc/host-peer.ts b/src/client/webrtc/host-peer.ts index e47d338..9b563ea 100644 --- a/src/client/webrtc/host-peer.ts +++ b/src/client/webrtc/host-peer.ts @@ -330,18 +330,12 @@ export class HostPeer { const effectiveVideoProfile = this.startupVideoProfilePending ? startupVideoProfile(profile) : profile; - const requestedVideo = - this.appliedVideoProfile === null || - !videoQualitySettingsEqual( - this.appliedVideoProfile, - effectiveVideoProfile, - ); - const requestedAudio = - !screenAudioQualityEqual(this.desiredProfile, profile) || - this.appliedAudioQuality !== - resolveScreenAudioQuality(profile.screenAudioQuality); + const changed = + !videoQualitySettingsEqual(this.desiredProfile, profile) || + !screenAudioQualityEqual(this.desiredProfile, profile); this.desiredProfile = profile; - const requestedRevision = ++this.profileRevision; + // Connection recovery replays the same intent, not a new user mutation. + const requestedRevision = changed ? ++this.profileRevision : this.profileRevision; return this.enqueueSenderMutation(async () => { const videoSender = this.videoSender; const audioSender = this.audioSender; @@ -367,14 +361,12 @@ export class HostPeer { } const updateVideo = this.connection.connectionState === "connected" && - (requestedVideo || - this.appliedVideoProfile === null || + (this.appliedVideoProfile === null || !videoQualitySettingsEqual( this.appliedVideoProfile, effectiveVideoProfile, )); const updateAudio = - requestedAudio || this.appliedAudioQuality !== resolveScreenAudioQuality(profile.screenAudioQuality); if (!updateVideo && !updateAudio) { @@ -508,6 +500,8 @@ export class HostPeer { this.appliedVideoProfile = null; this.appliedAudioQuality = null; this.appliedAudioSenderParameters = null; + } + if (replayProfile || !this.startupVideoProfilePending) { void this.updateProfile(this.desiredProfile); } } else if (this.connectedOnce && state !== "closed") { diff --git a/src/shared/protocol.ts b/src/shared/protocol.ts index 294071a..4c27273 100644 --- a/src/shared/protocol.ts +++ b/src/shared/protocol.ts @@ -7,7 +7,7 @@ import { NAT_TRAVERSAL_PATHS } from "./nat-candidate.js"; export const MAX_VIEWERS_PER_ROOM_LIMIT = 20; export const MAX_PARTICIPANTS_PER_ROOM_LIMIT = MAX_VIEWERS_PER_ROOM_LIMIT + 1; export const MAX_SIGNAL_BYTES = 64 * 1024; -export const SIGNALING_PROTOCOL = "screener-v21"; +export const SIGNALING_PROTOCOL = "screener-v22"; export const SIGNAL_CLOSE_CODES = { serviceRestart: 1012, sessionReplaced: 4001, @@ -17,7 +17,6 @@ export const SIGNAL_CLOSE_CODES = { } as const; export const ROOM_CODE_LENGTH = 4; export const MAX_MEDIA_ROUTE_REVISION = Number.MAX_SAFE_INTEGER; -export const MAX_SFU_TOKEN_LENGTH = 8 * 1024; export const MAX_ICE_SERVER_URLS = 8; export const MAX_NAT_PREDICTION_AUXILIARY_STUN_URLS = 2; export const MAX_VIEWER_QUALITY_EVIDENCE_BYTES = 2 * 1024; @@ -145,18 +144,6 @@ export const viewerPasswordSchema = z export const codeEntryPolicySchema = z.enum(["open", "private"]); export type CodeEntryPolicy = z.infer; -const liveKitWebSocketUrlSchema = z - .string() - .url() - .max(2048) - .refine((value) => { - try { - const protocol = new URL(value).protocol; - return protocol === "ws:" || protocol === "wss:"; - } catch { - return false; - } - }); export const roomCodeSchema = z .string() @@ -342,6 +329,43 @@ export const signalPayloadSchema = z.discriminatedUnion("kind", [ ]); export type SignalPayload = z.infer; +export const sfuMediaSchema = z.object({ + codec: z.enum(["h264", "vp8"]), + layers: z.array(z.object({ + rid: z.string().max(16), + width: z.number().int().min(1).max(8192), + height: z.number().int().min(1).max(8192), + bitrate: z.number().int().min(1).max(100_000_000), + }).strict()).min(1).max(3), + audio: z.boolean(), + audioBitrate: z.number().int().min(0).max(510_000), +}).strict().refine((media) => + (!media.audio || media.audioBitrate > 0) && + new Set(media.layers.map((layer) => layer.rid)).size === media.layers.length && + (media.layers.length === 1 || media.layers.every((layer) => layer.rid !== "")), +); +export type SfuMedia = z.infer; + +const sfuSignalMessageSchema = z.object({ + type: z.literal("sfu-signal"), + revision: z.number().int().min(0).max(Number.MAX_SAFE_INTEGER), + publicationGeneration: opaqueIdSchema, + connectionId: opaqueIdSchema, + kind: z.enum(["subscribe", "description", "candidate", "media", "layers"]), + description: sessionDescriptionSchema.optional(), + candidate: iceCandidateSchema.optional(), + media: sfuMediaSchema.optional(), + activeCount: z.number().int().min(0).max(3).optional(), +}).strict().refine((message) => { + if (message.kind === "layers") return message.activeCount !== undefined && !message.description && !message.candidate && !message.media; + if (message.activeCount !== undefined) return false; + if (message.kind === "media") return Boolean(message.media) && !message.description && !message.candidate; + if (message.kind === "subscribe") return !message.description && !message.candidate && !message.media; + if (message.kind === "candidate") return !message.description && !message.media; + return Boolean(message.description) && !message.candidate && (!message.media || message.description?.type === "offer"); +}); +export type SfuSignalMessage = z.infer; + export const preparedRouteCandidateSchema = z .object({ childPeerId: opaqueIdSchema, @@ -887,6 +911,7 @@ export const clientMessageSchema = z.union([ revision: mediaRouteRevisionSchema, }) .strict(), + sfuSignalMessageSchema, z.object({ type: z.literal("request-route-diagnostic") }).strict(), viewerQualityEvidenceMessageSchema, senderQualityEvidenceMessageSchema, @@ -966,7 +991,6 @@ const routeAuthenticatedShape = { routeRevision: mediaRouteRevisionSchema, routeAssignment: participantRouteAssignmentSchema, qualitySettings: qualitySettingsSchema, - sfuStandbyUrl: liveKitWebSocketUrlSchema.optional(), }; const authenticatedMessageSchema = z.union([ @@ -1054,10 +1078,11 @@ export const serverMessageSchema = z.union([ .object({ type: z.literal("sfu-config"), revision: mediaRouteRevisionSchema, - url: liveKitWebSocketUrlSchema, - token: z.string().min(1).max(MAX_SFU_TOKEN_LENGTH), + publicationGeneration: opaqueIdSchema, + connectionId: opaqueIdSchema, }) .strict(), + sfuSignalMessageSchema, z .object({ type: z.literal("quality-settings"), @@ -1069,7 +1094,6 @@ export const serverMessageSchema = z.union([ type: z.literal("route-policy"), shareGeneration: opaqueIdSchema, routePolicy: routePolicySchema, - sfuStandbyUrl: liveKitWebSocketUrlSchema.optional(), }) .strict(), z diff --git a/tests/browser-debug.test.ts b/tests/browser-debug.test.ts new file mode 100644 index 0000000..982b628 --- /dev/null +++ b/tests/browser-debug.test.ts @@ -0,0 +1,88 @@ +import { afterEach, describe, expect, it, vi } from "vitest"; + +function browser(search: string) { + const page = Object.assign(new EventTarget(), { + location: new URL( + `https://private.example/r/1234${search}#v=private-grant`, + ), + }) as unknown as Window; + vi.stubGlobal("window", page); + return page; +} + +afterEach(() => { + vi.unstubAllGlobals(); + vi.restoreAllMocks(); + vi.resetModules(); +}); + +describe("opt-in Browser diagnostics", () => { + it("is quiet without the explicit page flag", async () => { + const page = browser(""); + const logged = vi + .spyOn(console, "info") + .mockImplementation(() => undefined); + const debug = await import("../src/client/lib/debug"); + expect(debug.installBrowserDebug()).toBeUndefined(); + debug.debugEvent("capture", "requested", { resolution: "1080p" }); + debug.debugError("capture", "failed", new Error("private detail")); + expect(logged).not.toHaveBeenCalled(); + expect(page.__SCREENER_DEBUG__).toBeUndefined(); + }); + + it("keeps a bounded detached snapshot and excludes unapproved data and raw errors", async () => { + const page = browser("?debug=1&hostToken=private-token"); + vi.spyOn(console, "info").mockImplementation(() => undefined); + const debug = await import("../src/client/lib/debug"); + const cleanup = debug.installBrowserDebug(); + const captureError = new DOMException( + "candidate:1 1 UDP 1 192.0.2.1 1234 typ host", + "NotReadableError", + ); + debug.debugError("capture", "failed", captureError, { + resolution: "1080p", + maxFramerate: 30, + ...{ + hostToken: "private-token", + sdp: "private-sdp", + candidate: "private-candidate", + inviteUrl: "private-grant", + }, + }); + const report = page.__SCREENER_DEBUG__!.export(); + expect(report).toContain("NotReadableError"); + for (const secret of [ + "private-token", + "private-sdp", + "private-candidate", + "private-grant", + "192.0.2.1", + "private.example", + "/r/1234", + ]) + expect(report).not.toContain(secret); + const forgedError = new Error("private-message"); + forgedError.name = "private-token"; + debug.debugError("capture", "failed", forgedError); + expect(page.__SCREENER_DEBUG__!.events().at(-1)!.details.errorName).toBe( + "Error", + ); + for (let revision = 0; revision < 300; revision++) + debug.debugEvent("signal", "route", { revision }); + const events = page.__SCREENER_DEBUG__!.events(); + expect(events).toHaveLength(256); + expect(events[0]!.details.revision).toBe(44); + events[0]!.details.revision = -1; + expect(page.__SCREENER_DEBUG__!.events()[0]!.details.revision).toBe(44); + debug.debugEvent("signal", "route", { + state: "x".repeat(1000), + revision: Number.NaN, + }); + expect(page.__SCREENER_DEBUG__!.events().at(-1)!.details).toEqual({}); + page.__SCREENER_DEBUG__!.clear(); + expect(page.__SCREENER_DEBUG__!.events()).toEqual([]); + cleanup?.(); + page.dispatchEvent(new Event("error")); + expect(page.__SCREENER_DEBUG__!.events()).toEqual([]); + }); +}); diff --git a/tests/client-state.test.ts b/tests/client-state.test.ts index 285f6f3..f5be867 100644 --- a/tests/client-state.test.ts +++ b/tests/client-state.test.ts @@ -28,6 +28,7 @@ import { parseAppRoute, mergeAuthenticatedHostRoom, readHostRoom, + releaseHostRoom, readPreferredRoomId, readViewerGrant, readViewerRoute, @@ -58,6 +59,7 @@ import { } from "../src/client/webrtc/stats.ts"; afterEach(() => { + releaseHostRoom(); vi.useRealTimers(); vi.unstubAllGlobals(); setCopy({ lang: "zh", vis: false }); @@ -84,15 +86,15 @@ function routeAuthenticated( describe("browser-local display name", () => { it("localizes text defaults and uses visual emoji identities", () => { setCopy({ lang: "zh", vis: false }); - expect(defaultViewerDisplayName("viewer-abcdef", false)).toBe("观众"); - expect(defaultHostDisplayName("host-abcdef", false)).toBe("分享者 (abcdef)"); + expect(defaultViewerDisplayName(false)).toBe("观众"); + expect(defaultHostDisplayName(false)).toBe("分享者"); setCopy({ lang: "en", vis: false }); - expect(defaultViewerDisplayName("viewer-abcdef", false)).toBe("Viewer"); - expect(defaultHostDisplayName("host-abcdef", false)).toBe("Host (abcdef)"); + expect(defaultViewerDisplayName(false)).toBe("Viewer"); + expect(defaultHostDisplayName(false)).toBe("Host"); - expect(defaultViewerDisplayName("viewer-abcdef", true)).toBe("👤 (abcdef)"); - expect(defaultHostDisplayName("host-abcdef", true)).toBe("👑 (abcdef)"); + expect(defaultViewerDisplayName(true)).toBe("👤"); + expect(defaultHostDisplayName(true)).toBe("👑"); }); it("stores only the canonical preference and falls back when cleared", () => { @@ -284,6 +286,33 @@ describe("browser-local display name", () => { }); }); +function hostRoomStorage(lockState: "available" | "held" | "absent" = "available") { + const local = new Map(); + const session = new Map(); + const held = new Set(); + const storage = (values: Map) => ({ + getItem: (key: string) => values.get(key) ?? null, + setItem: (key: string, value: string) => values.set(key, value), + removeItem: (key: string) => values.delete(key), + }); + const request = vi.fn(async (name: string, _options: LockOptions, callback: (lock: Lock | null) => Promise) => { + if (lockState === "held" || held.has(name)) return callback(null); + held.add(name); + try { await callback({ name, mode: "exclusive" } as Lock); } + finally { held.delete(name); } + }); + vi.stubGlobal("window", { + localStorage: storage(local), sessionStorage: storage(session), + navigator: { locks: lockState === "absent" ? undefined : { request } }, + }); + return { local, session, held, request }; +} + +const storedHost = (roomId = "1234", hostToken = "a".repeat(32)) => ({ + roomId, hostToken, inviteUrl: `https://share.test/r/${roomId}`, + expiresAt: null, roomLeaseSeconds: 3_600, +}); + describe("client session identity", () => { it("works without secure-context-only crypto.randomUUID", () => { const values = new Map(); @@ -306,15 +335,8 @@ describe("client session identity", () => { expect(getStableClientId("viewer", "room-id-1234")).toBe(first); }); - it("persists one validated host room across browser sessions", () => { - const values = new Map(); - vi.stubGlobal("window", { - localStorage: { - getItem: (key: string) => values.get(key) ?? null, - setItem: (key: string, value: string) => values.set(key, value), - removeItem: (key: string) => values.delete(key), - }, - }); + it("persists a tab room and one validated origin resume hint without Viewer grants", async () => { + const { local: values, session } = hostRoomStorage(); const room = { roomId: "1234", hostToken: "a".repeat(32), @@ -324,9 +346,9 @@ describe("client session identity", () => { roomLeaseSeconds: 3_600, }; - writeHostRoom(room); + await writeHostRoom(room); - expect(readHostRoom()).toEqual({ + expect(await readHostRoom()).toEqual({ roomId: "1234", hostToken: "a".repeat(32), canonicalUrl: "https://share.test/r/1234", @@ -343,50 +365,28 @@ describe("client session identity", () => { }), ); expect([...values.values()].join(" ")).not.toContain(`${"b".repeat(21)}A`); + expect(session.get("screener:host-room:v1")).toBe(values.get("screener:host-room:v1")); clearHostRoom(); - expect(readHostRoom()).toBeNull(); + expect(await readHostRoom()).toBeNull(); }); - it("reclaims a deployed Host ownership record through the stable shape", () => { - const values = new Map([ - [ - "screener:host-room:v1", - JSON.stringify({ - roomId: "1234", - hostToken: "h".repeat(32), - inviteUrl: "https://share.test/r/1234", - expiresAt: null, - roomLeaseSeconds: 3_600, - }), - ], - ]); - vi.stubGlobal("window", { - localStorage: { - getItem: (key: string) => values.get(key) ?? null, - setItem: (key: string, value: string) => values.set(key, value), - removeItem: (key: string) => values.delete(key), - }, - }); - - expect(readHostRoom()).toEqual({ + it("restores an origin resume hint only after claiming its room", async () => { + const { local, session, request } = hostRoomStorage(); + local.set("screener:host-room:v1", JSON.stringify(storedHost("1234", "h".repeat(32)))); + expect(await readHostRoom()).toEqual({ roomId: "1234", hostToken: "h".repeat(32), canonicalUrl: "https://share.test/r/1234", expiresAt: null, roomLeaseSeconds: 3_600, }); + expect(JSON.parse(session.get("screener:host-room:v1")!)).toEqual(JSON.parse(local.get("screener:host-room:v1")!)); + expect(request).toHaveBeenCalledWith(expect.stringMatching(/^screener:host-room:1234:[a-f0-9]{64}$/), { ifAvailable: true }, expect.any(Function)); + expect(request.mock.calls[0]![0]).not.toContain("h".repeat(32)); }); - it("keeps expired host records for server judgment and discards malformed records", () => { - const values = new Map(); - const removeItem = vi.fn((key: string) => values.delete(key)); - vi.stubGlobal("window", { - localStorage: { - getItem: (key: string) => values.get(key) ?? null, - setItem: (key: string, value: string) => values.set(key, value), - removeItem, - }, - }); + it("keeps expired host records for server judgment and discards malformed records", async () => { + const { local: values, session } = hostRoomStorage(); const expired = { roomId: "1234", hostToken: "b".repeat(32), @@ -395,13 +395,123 @@ describe("client session identity", () => { roomLeaseSeconds: 3_600, }; - writeHostRoom(expired); - expect(readHostRoom()).toMatchObject({ roomId: "1234" }); - expect(removeItem).not.toHaveBeenCalled(); - + await writeHostRoom(expired); + expect(await readHostRoom()).toMatchObject({ roomId: "1234" }); + releaseHostRoom(); + session.delete("screener:host-room:v1"); values.set("screener:host-room:v1", "not-json"); - expect(readHostRoom()).toBeNull(); - expect(removeItem).toHaveBeenCalledOnce(); + expect(await readHostRoom()).toBeNull(); + expect(values.has("screener:host-room:v1")).toBe(false); + }); + + it("rejects a copied tab room while another tab holds it without deleting that hint", async () => { + const { local, session } = hostRoomStorage("held"); + const saved = JSON.stringify(storedHost()); + local.set("screener:host-room:v1", saved); + session.set("screener:host-room:v1", saved); + expect(await readHostRoom()).toBeNull(); + expect(session.has("screener:host-room:v1")).toBe(false); + expect(local.get("screener:host-room:v1")).toBe(saved); + }); + + it("prefers this tab's room over another tab's latest hint and preserves that hint when forgotten", async () => { + const { local, session } = hostRoomStorage(); + session.set("screener:host-room:v1", JSON.stringify(storedHost())); + const other = JSON.stringify(storedHost("5678", "b".repeat(32))); + local.set("screener:host-room:v1", other); + expect(await readHostRoom()).toMatchObject({ roomId: "1234" }); + clearHostRoom(); + expect(session.has("screener:host-room:v1")).toBe(false); + expect(local.get("screener:host-room:v1")).toBe(other); + }); + + it("retains both records on unmount, reclaims on refresh, and releases the old claim on rotation", async () => { + const { local, session, held, request } = hostRoomStorage(); + const room = storedHost(); + await writeHostRoom({ ...room, codeEntryPolicy: "open" }); + const stored = local.get("screener:host-room:v1"); + releaseHostRoom(); + expect(session.get("screener:host-room:v1")).toBe(stored); + expect(local.get("screener:host-room:v1")).toBe(stored); + expect(await readHostRoom()).toMatchObject({ roomId: "1234" }); + expect(request).toHaveBeenCalledTimes(2); + await writeHostRoom({ ...storedHost("5678", "b".repeat(32)), codeEntryPolicy: "open" }); + expect(held.size).toBe(1); + expect([...held][0]).toContain(":5678:"); + }); + + it("coalesces concurrent restoration and preserves records when an unmounted restore is cancelled", async () => { + const { local, session, request } = hostRoomStorage(); + local.set("screener:host-room:v1", JSON.stringify(storedHost())); + const first = await Promise.all([readHostRoom(), readHostRoom()]); + expect(first[0]).toEqual(first[1]); + expect(request).toHaveBeenCalledOnce(); + releaseHostRoom(); + const cancelled = readHostRoom(); + releaseHostRoom(); + expect(await cancelled).toBeNull(); + expect(session.has("screener:host-room:v1")).toBe(true); + expect(await readHostRoom()).toMatchObject({ roomId: "1234" }); + }); + + it("does not reuse the origin hint without Web Locks but keeps same-tab refresh authority", async () => { + const { local, session } = hostRoomStorage("absent"); + const saved = JSON.stringify(storedHost()); + local.set("screener:host-room:v1", saved); + expect(await readHostRoom()).toBeNull(); + session.set("screener:host-room:v1", saved); + expect(await readHostRoom()).toMatchObject({ roomId: "1234" }); + releaseHostRoom(); + expect(await readHostRoom()).toMatchObject({ roomId: "1234" }); + }); + + it("does not erase the resume hint when the server replaces this Host session", async () => { + const { local, session } = hostRoomStorage(); + await writeHostRoom({ ...storedHost(), codeEntryPolicy: "open" }); + const saved = local.get("screener:host-room:v1"); + clearHostRoom(true); + expect(session.has("screener:host-room:v1")).toBe(false); + expect(local.get("screener:host-room:v1")).toBe(saved); + }); + + it("keeps same-tab authority when Web Locks are denied, without restoring the origin hint", async () => { + const { local, session, request } = hostRoomStorage(); + request.mockRejectedValue(new DOMException("Blocked", "SecurityError")); + const saved = JSON.stringify(storedHost()); + local.set("screener:host-room:v1", saved); + expect(await readHostRoom()).toBeNull(); + session.set("screener:host-room:v1", saved); + expect(await readHostRoom()).toMatchObject({ roomId: "1234" }); + expect(session.get("screener:host-room:v1")).toBeTruthy(); + }); + + it("does not persist an obsolete creation and gives a reused code a distinct incarnation claim", async () => { + const { local, held, request } = hostRoomStorage(); + let owns = true; + const cancelled = writeHostRoom({ ...storedHost(), codeEntryPolicy: "open" }, () => owns); + owns = false; + expect(await cancelled).toBe(false); + expect(local.size).toBe(0); + await writeHostRoom({ ...storedHost(), codeEntryPolicy: "open" }); + const first = request.mock.calls.at(-1)![0]; + await writeHostRoom({ ...storedHost("1234", "b".repeat(32)), codeEntryPolicy: "open" }); + expect(request.mock.calls.at(-1)![0]).not.toBe(first); + expect(held.size).toBe(1); + }); + + it("retains the tab claim when a share's metadata update becomes obsolete", async () => { + const { held, request, local } = hostRoomStorage(); + const room = { ...storedHost(), codeEntryPolicy: "open" as const }; + await writeHostRoom(room); + const saved = local.get("screener:host-room:v1"); + let owns = true; + const writing = writeHostRoom({ ...room, expiresAt: "2026-09-09T00:00:00.000Z" }, () => owns); + owns = false; + expect(await writing).toBe(false); + expect(held.size).toBe(1); + expect(local.get("screener:host-room:v1")).toBe(saved); + expect(await readHostRoom()).toMatchObject({ roomId: "1234", expiresAt: null }); + expect(request).toHaveBeenCalledOnce(); }); it("consumes a Viewer grant fragment once into room-scoped session storage", () => { diff --git a/tests/fixtures/wire-samples.json b/tests/fixtures/wire-samples.json index 553090e..27baee3 100644 --- a/tests/fixtures/wire-samples.json +++ b/tests/fixtures/wire-samples.json @@ -1,12 +1,11 @@ { "constants": { - "signalingProtocol": "screener-v21", + "signalingProtocol": "screener-v22", "maxViewersPerRoomLimit": 20, "maxParticipantsPerRoomLimit": 21, "maxSignalBytes": 65536, "roomCodeLength": 4, "maxMediaRouteRevision": 9007199254740991, - "maxSfuTokenLength": 8192, "maxIceServerUrls": 8, "maxNatPredictionAuxiliaryStunUrls": 2, "maxViewerQualityEvidenceBytes": 2048, @@ -42,79 +41,79 @@ { "name": "authenticate-host-minimal", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v21\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v22\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\"}", "valid": true }, { "name": "authenticate-host-full", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v21\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\",\"shareGeneration\":\"share_generation_12345678\",\"sharingPaused\":false,\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"screenAudioQuality\":\"music\"},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"viewerPresence\":true,\"displayName\":\"小明\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v22\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\",\"shareGeneration\":\"share_generation_12345678\",\"sharingPaused\":false,\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"screenAudioQuality\":\"music\"},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"viewerPresence\":true,\"displayName\":\"小明\"}", "valid": true }, { "name": "authenticate-host-carries-viewer-grant", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v21\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\",\"viewerGrant\":\"bbbbbbbbbbbbbbbbbbbbbg\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v22\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\",\"viewerGrant\":\"bbbbbbbbbbbbbbbbbbbbbg\"}", "valid": false }, { "name": "authenticate-host-carries-viewer-password", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v21\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\",\"viewerPassword\":\"easy-password\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v22\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\",\"viewerPassword\":\"easy-password\"}", "valid": false }, { "name": "authenticate-host-short-token", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v21\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"short\",\"clientId\":\"client_12345678\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v22\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"short\",\"clientId\":\"client_12345678\"}", "valid": false }, { "name": "authenticate-host-unknown-key", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v21\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\",\"admin\":true}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v22\",\"roomId\":\"1234\",\"role\":\"host\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\",\"clientId\":\"client_12345678\",\"admin\":true}", "valid": false }, { "name": "authenticate-viewer-minimal", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v21\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v22\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\"}", "valid": true }, { "name": "authenticate-viewer-full", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v21\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"viewerGrant\":\"bbbbbbbbbbbbbbbbbbbbbg\",\"viewerPassword\":\"easy-password\",\"displayName\":\"Café 朋友\",\"viewerPresence\":true}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v22\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"viewerGrant\":\"bbbbbbbbbbbbbbbbbbbbbg\",\"viewerPassword\":\"easy-password\",\"displayName\":\"Café 朋友\",\"viewerPresence\":true}", "valid": true }, { "name": "authenticate-viewer-carries-host-token", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v21\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v22\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"token\":\"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\"}", "valid": false }, { "name": "authenticate-viewer-carries-route-policy", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v21\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false}}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v22\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false}}", "valid": false }, { "name": "authenticate-viewer-malformed-grant", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v21\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"viewerGrant\":\"bbbbbbbbbbbbbbbbbbbbb\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v22\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"viewerGrant\":\"bbbbbbbbbbbbbbbbbbbbb\"}", "valid": false }, { "name": "authenticate-viewer-grant-bad-final-character", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v21\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"viewerGrant\":\"bbbbbbbbbbbbbbbbbbbbbb\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v22\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"viewerGrant\":\"bbbbbbbbbbbbbbbbbbbbbb\"}", "valid": false }, { "name": "authenticate-viewer-unknown-capabilities", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v21\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"capabilities\":{\"peerIceTurn\":true}}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v22\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"capabilities\":{\"peerIceTurn\":true}}", "valid": false }, { @@ -132,55 +131,55 @@ { "name": "authenticate-viewer-leading-zero-room", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v21\",\"roomId\":\"0123\",\"role\":\"viewer\",\"clientId\":\"client_12345678\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v22\",\"roomId\":\"0123\",\"role\":\"viewer\",\"clientId\":\"client_12345678\"}", "valid": false }, { "name": "authenticate-viewer-long-room", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v21\",\"roomId\":\"11111\",\"role\":\"viewer\",\"clientId\":\"client_12345678\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v22\",\"roomId\":\"11111\",\"role\":\"viewer\",\"clientId\":\"client_12345678\"}", "valid": false }, { "name": "authenticate-viewer-noncanonical-display-name", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v21\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":\" Café \"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v22\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":\" Café \"}", "valid": false }, { "name": "authenticate-viewer-display-name-too-long", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v21\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":\"名名名名名名名名名名名名名名名名名名名名名名名名名\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v22\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":\"名名名名名名名名名名名名名名名名名名名名名名名名名\"}", "valid": false }, { "name": "authenticate-viewer-display-name-at-limit", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v21\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":\"名名名名名名名名名名名名名名名名名名名名名名名名\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v22\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":\"名名名名名名名名名名名名名名名名名名名名名名名名\"}", "valid": true }, { "name": "authenticate-viewer-display-name-bidi-override", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v21\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":\"a‮b\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v22\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":\"a‮b\"}", "valid": false }, { "name": "authenticate-viewer-display-name-newline", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v21\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":\"a\\nb\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v22\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":\"a\\nb\"}", "valid": false }, { "name": "authenticate-viewer-null-display-name", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v21\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":null}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v22\",\"roomId\":\"1234\",\"role\":\"viewer\",\"clientId\":\"client_12345678\",\"displayName\":null}", "valid": false }, { "name": "authenticate-unknown-role", "schema": "client", - "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v21\",\"roomId\":\"1234\",\"role\":\"relay\",\"clientId\":\"client_12345678\"}", + "json": "{\"type\":\"authenticate\",\"protocol\":\"screener-v22\",\"roomId\":\"1234\",\"role\":\"relay\",\"clientId\":\"client_12345678\"}", "valid": false }, { @@ -776,24 +775,54 @@ "schema": "roomAccessUpdateRequest", "json": "{\"action\":\"disable-room\"}", "valid": false + }, + { + "name": "sfu-publisher-offer", + "schema": "client", + "json": "{\"type\":\"sfu-signal\",\"revision\":9,\"publicationGeneration\":\"publication_12345678\",\"connectionId\":\"connection_12345678\",\"kind\":\"description\",\"description\":{\"type\":\"offer\",\"sdp\":\"v=0\\\\r\\\\n\"},\"media\":{\"codec\":\"vp8\",\"layers\":[{\"rid\":\"q\",\"width\":640,\"height\":360,\"bitrate\":500000},{\"rid\":\"h\",\"width\":1280,\"height\":720,\"bitrate\":2000000}],\"audio\":false,\"audioBitrate\":128000}}", + "valid": true + }, + { + "name": "sfu-ice-end", + "schema": "client", + "json": "{\"type\":\"sfu-signal\",\"revision\":9,\"publicationGeneration\":\"publication_12345678\",\"connectionId\":\"connection_12345678\",\"kind\":\"candidate\"}", + "valid": true + }, + { + "name": "sfu-invalid-layers", + "schema": "client", + "json": "{\"type\":\"sfu-signal\",\"revision\":9,\"publicationGeneration\":\"publication_12345678\",\"connectionId\":\"connection_12345678\",\"kind\":\"description\",\"description\":{\"type\":\"offer\",\"sdp\":\"v=0\\\\r\\\\n\"},\"media\":{\"codec\":\"vp8\",\"layers\":[],\"audio\":false,\"audioBitrate\":128000}}", + "valid": false + }, + { + "name": "sfu-invalid-media-role", + "schema": "client", + "json": "{\"type\":\"sfu-signal\",\"revision\":9,\"publicationGeneration\":\"publication_12345678\",\"connectionId\":\"connection_12345678\",\"kind\":\"description\",\"description\":{\"type\":\"answer\",\"sdp\":\"v=0\"},\"media\":{\"codec\":\"vp8\",\"layers\":[{\"rid\":\"q\",\"width\":640,\"height\":360,\"bitrate\":500000},{\"rid\":\"h\",\"width\":1280,\"height\":720,\"bitrate\":2000000}],\"audio\":false,\"audioBitrate\":128000}}", + "valid": false + }, + { + "name": "sfu-null-description", + "schema": "client", + "json": "{\"type\":\"sfu-signal\",\"revision\":9,\"publicationGeneration\":\"publication_12345678\",\"connectionId\":\"connection_12345678\",\"kind\":\"candidate\",\"description\":null}", + "valid": false } ], "serverMessages": [ { "name": "authenticated-host-minimal", - "json": "{\"type\":\"authenticated\",\"protocol\":\"screener-v21\",\"peerId\":\"host_12345678\",\"roomExpiresAt\":\"2026-08-18T18:00:00.000Z\",\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"connectionId\":null,\"iceConfig\":{\"iceServers\":[],\"natPredictionStunUrls\":[]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"none\"},\"childPeerIds\":[],\"sfuPublicationGeneration\":null},\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"screenAudioQuality\":\"music\"},\"role\":\"host\",\"viewerPasswordEnabled\":false}" + "json": "{\"type\":\"authenticated\",\"protocol\":\"screener-v22\",\"peerId\":\"host_12345678\",\"roomExpiresAt\":\"2026-08-18T18:00:00.000Z\",\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"connectionId\":null,\"iceConfig\":{\"iceServers\":[],\"natPredictionStunUrls\":[]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"none\"},\"childPeerIds\":[],\"sfuPublicationGeneration\":null},\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"screenAudioQuality\":\"music\"},\"role\":\"host\",\"viewerPasswordEnabled\":false}" }, { "name": "authenticated-host-full", - "json": "{\"type\":\"authenticated\",\"protocol\":\"screener-v21\",\"peerId\":\"host_12345678\",\"roomExpiresAt\":null,\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"connectionId\":\"connection_12345678\",\"iceConfig\":{\"iceServers\":[{\"urls\":\"stun:stun.example.test:3478\"},{\"urls\":[\"stun:stun-1.example.test:3478\",\"stun:[2001:db8::1]:3478\"]}],\"natPredictionStunUrls\":[\"stun:nat.example.test:3478\"]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"sfu\"},\"childPeerIds\":[],\"sfuPublicationGeneration\":\"generation_12345678\"},\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"screenAudioQuality\":\"music\"},\"sfuStandbyUrl\":\"wss://sfu.example.test\",\"role\":\"host\",\"viewerPasswordEnabled\":false}" + "json": "{\"type\":\"authenticated\",\"protocol\":\"screener-v22\",\"peerId\":\"host_12345678\",\"roomExpiresAt\":null,\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"connectionId\":\"connection_12345678\",\"iceConfig\":{\"iceServers\":[{\"urls\":\"stun:stun.example.test:3478\"},{\"urls\":[\"stun:stun-1.example.test:3478\",\"stun:[2001:db8::1]:3478\"]}],\"natPredictionStunUrls\":[\"stun:nat.example.test:3478\"]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"sfu\"},\"childPeerIds\":[],\"sfuPublicationGeneration\":\"generation_12345678\"},\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"screenAudioQuality\":\"music\"},\"role\":\"host\",\"viewerPasswordEnabled\":false}" }, { "name": "authenticated-viewer-minimal", - "json": "{\"type\":\"authenticated\",\"protocol\":\"screener-v21\",\"peerId\":\"viewer_12345678\",\"roomExpiresAt\":\"2026-08-18T18:00:00.000Z\",\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"hostPaused\":false,\"connectionId\":null,\"iceConfig\":{\"iceServers\":[],\"natPredictionStunUrls\":[]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"none\"},\"childPeerIds\":[],\"sfuPublicationGeneration\":null},\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"screenAudioQuality\":\"music\"},\"role\":\"viewer\"}" + "json": "{\"type\":\"authenticated\",\"protocol\":\"screener-v22\",\"peerId\":\"viewer_12345678\",\"roomExpiresAt\":\"2026-08-18T18:00:00.000Z\",\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"hostPaused\":false,\"connectionId\":null,\"iceConfig\":{\"iceServers\":[],\"natPredictionStunUrls\":[]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"none\"},\"childPeerIds\":[],\"sfuPublicationGeneration\":null},\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"screenAudioQuality\":\"music\"},\"role\":\"viewer\"}" }, { "name": "authenticated-viewer-full", - "json": "{\"type\":\"authenticated\",\"protocol\":\"screener-v21\",\"peerId\":\"viewer_12345678\",\"roomExpiresAt\":\"2026-08-18T18:00:00.000Z\",\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"hostPaused\":true,\"connectionId\":\"connection_12345678\",\"iceConfig\":{\"iceServers\":[],\"natPredictionStunUrls\":[]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"peer\",\"peerId\":\"parent_12345678\"},\"childPeerIds\":[\"child_12345678\",\"child_87654321\"],\"sfuPublicationGeneration\":null},\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"screenAudioQuality\":\"music\"},\"sfuStandbyUrl\":\"wss://sfu.example.test\",\"role\":\"viewer\"}" + "json": "{\"type\":\"authenticated\",\"protocol\":\"screener-v22\",\"peerId\":\"viewer_12345678\",\"roomExpiresAt\":\"2026-08-18T18:00:00.000Z\",\"maxViewers\":8,\"endpointMediaCopyCapacity\":2,\"hostOnline\":true,\"hostPaused\":true,\"connectionId\":\"connection_12345678\",\"iceConfig\":{\"iceServers\":[],\"natPredictionStunUrls\":[]},\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"codeEntryPolicy\":\"open\",\"viewerAuthorizationGeneration\":\"viewer_generation_12345678\",\"mediaMode\":\"peer-assisted\",\"shareGeneration\":\"share_generation_12345678\",\"routeRevision\":0,\"routeAssignment\":{\"upstream\":{\"kind\":\"peer\",\"peerId\":\"parent_12345678\"},\"childPeerIds\":[\"child_12345678\",\"child_87654321\"],\"sfuPublicationGeneration\":null},\"qualitySettings\":{\"resolution\":\"1080p\",\"maxFramerate\":60,\"maxBitrate\":8000000,\"degradationPreference\":\"maintain-resolution\",\"screenAudioQuality\":\"music\"},\"role\":\"viewer\"}" }, { "name": "signaling-challenge-response", @@ -845,7 +874,7 @@ }, { "name": "sfu-config", - "json": "{\"type\":\"sfu-config\",\"revision\":9,\"url\":\"wss://sfu.example.test\",\"token\":\"header.payload.signature\"}" + "json": "{\"type\":\"sfu-config\",\"revision\":9,\"publicationGeneration\":\"publication_12345678\",\"connectionId\":\"connection_12345678\"}" }, { "name": "quality-settings", @@ -857,7 +886,7 @@ }, { "name": "route-policy", - "json": "{\"type\":\"route-policy\",\"shareGeneration\":\"share_generation_12345678\",\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false},\"sfuStandbyUrl\":\"wss://sfu.example.test\"}" + "json": "{\"type\":\"route-policy\",\"shareGeneration\":\"share_generation_12345678\",\"routePolicy\":{\"peerOnly\":false,\"topologyOptimization\":true,\"natPrediction\":false}}" }, { "name": "route-policy-peer-only", @@ -906,6 +935,14 @@ { "name": "error", "json": "{\"type\":\"error\",\"code\":\"ROOM_FULL\",\"message\":\"Room is full\"}" + }, + { + "name": "sfu-subscriber-offer", + "json": "{\"type\":\"sfu-signal\",\"revision\":9,\"publicationGeneration\":\"publication_12345678\",\"connectionId\":\"connection_12345678\",\"kind\":\"description\",\"description\":{\"type\":\"offer\",\"sdp\":\"v=0\\\\r\\\\n\"}}" + }, + { + "name": "sfu-layer-demand", + "json": "{\"type\":\"sfu-signal\",\"revision\":9,\"publicationGeneration\":\"publication_12345678\",\"connectionId\":\"connection_12345678\",\"kind\":\"layers\",\"activeCount\":0}" } ] } diff --git a/tests/host-page-notice.test.ts b/tests/host-page-notice.test.ts index 0669fbe..59ca21d 100644 --- a/tests/host-page-notice.test.ts +++ b/tests/host-page-notice.test.ts @@ -1,6 +1,7 @@ import { describe, expect, it } from "vitest"; import { setCopy } from "../src/client/ui/copy.ts"; +import { NativeMediaBridgeError } from "../src/client/native/media-bridge"; import { hostActionErrorNotice, hostServerErrorNotice, @@ -11,6 +12,11 @@ import { setCopy({ lang: "zh" }); describe("host error notices", () => { + it("classifies the native preview bridge as a connection failure", () => { + expect(hostActionErrorNotice(new NativeMediaBridgeError("private detail"), "capture")) + .toBe(hostActionErrorNotice(new Error("private detail"), "connection")); + }); + it("maps every server error code without exposing server text", () => { const notices = { AUTH_REQUIRED: "站点访问已失效，请重新验证", diff --git a/tests/host-peer-tracks.test.ts b/tests/host-peer-tracks.test.ts index 6547922..4489c95 100644 --- a/tests/host-peer-tracks.test.ts +++ b/tests/host-peer-tracks.test.ts @@ -1159,6 +1159,40 @@ describe("HostPeer source replacement", () => { expect(videoSender.appliedMaxBitrates).toEqual([3_000_000, 8_000_000]); }); + it.each([false, true])("applies connecting profile changes without canceling an in-flight constraint: %s", async (holdConstraints) => { + const peer = createPeer(createStream(createTrack("video", "video"), createTrack("audio", "audio"))); + await expect(peer.start()).resolves.toBe(true); + const connection = FakePeerConnection.latest!; + const sender = connection.senders[0]!; + await peer.acceptSignal({ + kind: "description", connectionId: peer.connectionId, + description: { type: "answer", sdp: "test-answer" }, + }); + await completeVideoStartup(connection, 5); + const before = sender.setParameters.mock.calls.length; + let release: (() => void) | undefined; + if (holdConstraints) { + vi.mocked(sender.track!.applyConstraints).mockImplementationOnce(() => + new Promise((resolve) => { release = resolve; }), + ); + } + const changing = holdConstraints + ? peer.updateCaptureProfile(QUALITY_PROFILES["1080p60"]) + : peer.updateProfile(QUALITY_PROFILES["1080p60"]); + if (holdConstraints) { + await vi.waitFor(() => expect(release).toBeTypeOf("function")); + } else { + await expect(changing).resolves.toBe(true); + } + expect(sender.setParameters).toHaveBeenCalledTimes(before); + connection.connectionState = "connected"; + connection.dispatchEvent(new Event("connectionstatechange")); + release?.(); + await expect(changing).resolves.toBe(true); + await vi.waitFor(() => expect(sender.appliedMaxBitrates.at(-1)).toBe(8_000_000)); + expect(sender.setParameters).toHaveBeenCalledTimes(before + 1); + }); + it("replays the desired profile only after a connected peer reconnects", async () => { const peer = createPeer( createStream( diff --git a/tests/host-quality-ownership.test.ts b/tests/host-quality-ownership.test.ts new file mode 100644 index 0000000..c941f7d --- /dev/null +++ b/tests/host-quality-ownership.test.ts @@ -0,0 +1,197 @@ +import { readFileSync } from "node:fs"; +import { createContext, runInContext } from "node:vm"; +import ts from "typescript"; +import { describe, expect, it, vi } from "vitest"; +import { + QUALITY_PROFILES, qualitySettingsEqual, qualitySettingsLabel, + resolveScreenAudioQuality, videoQualitySettingsEqual, + type QualitySettings, +} from "../src/client/media/quality"; +import { NativeSenderPeer } from "../src/client/native/native-sender-peer"; + +// Exercise the actual page owners without mounting capture hardware or a Browser. +const source = ts.createSourceFile("HostPage.tsx", readFileSync( + new URL("../src/client/pages/HostPage.tsx", import.meta.url), "utf8", +), ts.ScriptTarget.Latest, true, ts.ScriptKind.TSX); +const owners = new Set(["changeQuality", "commitQuality", "handleSignalMessage", + "switchNativeSource", "finishSourceSwitch", "recoverBrowserFanout", "disposeNativeShare"]); +const functions: string[] = []; +function collect(node: ts.Node): void { + if (ts.isFunctionDeclaration(node) && node.name && owners.has(node.name.text)) { + functions.push(node.getText(source)); + owners.delete(node.name.text); + } + ts.forEachChild(node, collect); +} +collect(source); +if (owners.size) throw new Error(`Missing Host owner: ${[...owners].join(", ")}`); +const executable = ts.transpileModule(functions.join("\n"), { + compilerOptions: { target: ts.ScriptTarget.ES2022 }, +}).outputText; + +const original = QUALITY_PROFILES["1080p30"]; +const lower = QUALITY_PROFILES["720p30"]; +function ref(current: T) { return { current }; } +function deferred() { + let resolve!: (value: T) => void; + const promise = new Promise((next) => { resolve = next; }); + return { promise, resolve }; +} +function ingress() { + return { updateProfile: vi.fn(async (_profile: QualitySettings) => true), dispose: vi.fn() }; +} + +function fixture() { + let physical: QualitySettings = original; + const track = { stop: vi.fn() }; + const stream = { getTracks: () => [track] }; + const peer = { updateCaptureProfile: vi.fn(async () => true), updateProfile: vi.fn(async () => true) }; + const client = { updateShare: vi.fn(async () => undefined), replaceShareSource: vi.fn(async () => undefined), + stopReceive: vi.fn(async () => undefined), stopShare: vi.fn(async () => undefined) }; + const route = { updateProfile: vi.fn(async () => true), resyncAuthoritative: vi.fn(async (): Promise => undefined) }; + const state = { + phase: "live", qualitySettingsRef: ref(original), advancedQualityRef: ref(original), + qualityChangeRef: ref(null), pendingQualityChangeRef: ref(null), + activeGenerationRef: ref(1), streamRef: ref(stream), sourceSwitchRef: ref(null), + nativeModeRef: ref(false), nativeClientRef: ref(client), nativeShareGenerationRef: ref("share"), + nativeMediaIngressRef: ref | null>(null), nativeMediaBridgeRef: ref(null), + nativeEventCleanupRef: ref(null), nativeShareCleanupRef: ref(Promise.resolve()), + peersRef: ref(new Map([["viewer", peer]])), hostProvisionalChildRef: ref(null), hostSfuRouteRef: ref(route), + signalRef: ref({ setHostQualitySettings: vi.fn(), send: vi.fn() }), + setQualitySettings: vi.fn(), setAdvancedQuality: vi.fn(), setChangingQuality: vi.fn(), + setNotice: vi.fn(), setNoticeError: vi.fn(), setDetails: vi.fn(), setNativeActive: vi.fn(), + applyCaptureProfile: vi.fn(async (_stream: unknown, profile: QualitySettings) => { physical = profile; }), + captureDetails: () => ({}), nativeCaptureDetails: () => ({}), + qualitySettingsEqual, videoQualitySettingsEqual, resolveScreenAudioQuality, qualitySettingsLabel, + say: (key: string) => key, syncHostSfuQualityWarning: () => null, + NativeSenderPeer, discardPreparedHostChild: vi.fn(), removePeer: vi.fn(), startPeer: vi.fn(async () => undefined), + nativePreviewTailRef: ref(Promise.resolve()), setSwitchingSource: vi.fn(), + invalidateSenderQualityEvidence: vi.fn(), routePolicyRef: ref({ topologyOptimization: false }), + sourceSwitchNotice: () => "source-switch-result", hostPeerIdRef: ref(null), endpointMediaCopyCapacityRef: ref(2), + creationProfileRef: ref({ roomPassword: null }), saveCreationProfile: vi.fn(), setCreationProfile: vi.fn(), + setViewerPasswordEnabled: vi.fn(), setViewerPasswordDraft: vi.fn(), setViewerPasswordVisible: vi.fn(), + setMaxViewers: vi.fn(), setRoutePolicy: vi.fn(), setRoom: vi.fn(), activeRouteRevisionRef: ref(1), + ensureHostSfuRoute: () => route, + }; + const context = createContext({ ...state, isCurrentGeneration: (generation: number) => state.activeGenerationRef.current === generation }); + runInContext(executable, context); + return { ...state, peer, client, route, track, physical: () => physical, + change: context.changeQuality as (profile: QualitySettings) => Promise, + reauthenticate: () => context.handleSignalMessage({ type: "authenticated", role: "host", qualitySettings: original, + routePolicy: {}, routeRevision: 1 }, 1, { roomId: "room" }, true, null), + switchSource: () => context.switchNativeSource(client, {}, false, {}), + }; +} + +describe("Host quality ownership", () => { + it.each([false, "reject"])("keeps applied Browser capture after optional ingress fails: %s", async (failure) => { + const current = fixture(); + const optional = ingress(); + current.nativeMediaIngressRef.current = optional; + optional.updateProfile.mockImplementation(async () => { + expect(current.qualitySettingsRef.current).toEqual(lower); + expect(current.signalRef.current.setHostQualitySettings).toHaveBeenCalledWith(lower); + if (failure === "reject") throw new Error("ingress failed"); + return false; + }); + await current.change(lower); + expect(current.physical()).toEqual(lower); + expect(current.qualitySettingsRef.current).toEqual(lower); + expect(current.advancedQualityRef.current).toEqual(lower); + expect(current.peer.updateCaptureProfile).toHaveBeenCalledWith(lower); + expect(optional.dispose).toHaveBeenCalledOnce(); + expect(current.client.stopReceive).toHaveBeenCalledWith("share"); + expect(current.track.stop).not.toHaveBeenCalled(); + expect(current.setNoticeError).not.toHaveBeenCalled(); + }); + + it("retains the committed settings when Browser capture constraints reject", async () => { + const current = fixture(); + const optional = ingress(); + current.nativeMediaIngressRef.current = optional; + current.applyCaptureProfile.mockRejectedValue(new Error("capture rejected")); + await current.change(lower); + expect(current.physical()).toEqual(original); + expect(current.qualitySettingsRef.current).toEqual(original); + expect(current.advancedQualityRef.current).toEqual(original); + expect(optional.updateProfile).not.toHaveBeenCalled(); + expect(current.signalRef.current.setHostQualitySettings).not.toHaveBeenCalled(); + }); + + it("does not commit a capture completion after its share has ended", async () => { + const current = fixture(); + const pending = deferred(); + current.applyCaptureProfile.mockReturnValue(pending.promise); + const changing = current.change(lower); + current.activeGenerationRef.current = null; + current.qualityChangeRef.current = null; + pending.resolve(); + await changing; + expect(current.qualitySettingsRef.current).toEqual(original); + expect(current.signalRef.current.setHostQualitySettings).not.toHaveBeenCalled(); + }); + + it.each(["replacement", "ended"])("does not retire another ingress after an old update settles: %s", async (change) => { + const current = fixture(); + const old = ingress(), replacement = ingress(), pending = deferred(); + old.updateProfile.mockReturnValue(pending.promise); + current.nativeMediaIngressRef.current = old; + const changing = current.change(lower); + await vi.waitFor(() => expect(old.updateProfile).toHaveBeenCalled()); + current.nativeMediaIngressRef.current = replacement; + if (change === "ended") { + current.activeGenerationRef.current = null; + current.qualityChangeRef.current = null; + } + pending.resolve(false); + await changing; + expect(old.dispose).not.toHaveBeenCalled(); + expect(replacement.dispose).not.toHaveBeenCalled(); + expect(current.nativeMediaIngressRef.current).toBe(replacement); + if (change === "ended") expect(current.peer.updateCaptureProfile).not.toHaveBeenCalled(); + }); + + it.each([false, true])("preserves an active draft through reauthentication and commits its queued result: %s", async (queued) => { + const current = fixture(); + const pending = deferred(); + current.applyCaptureProfile.mockReturnValueOnce(pending.promise); + const changing = current.change(lower); + const latest = queued ? QUALITY_PROFILES["1080p60"] : lower; + if (queued) await current.change(latest); + current.reauthenticate(); + expect(current.qualitySettingsRef.current).toEqual(original); + expect(current.advancedQualityRef.current).toEqual(latest); + pending.resolve(); + await changing; + await vi.waitFor(() => expect(current.qualitySettingsRef.current).toEqual(latest)); + expect(current.advancedQualityRef.current).toEqual(latest); + expect(current.pendingQualityChangeRef.current).toBeNull(); + }); + + it("does not overwrite a new notice after a stale Native source-switch SFU update", async () => { + const current = fixture(); + const pending = deferred(); + current.route.updateProfile.mockReturnValue(pending.promise); + const switching = current.switchSource(); + await vi.waitFor(() => expect(current.route.updateProfile).toHaveBeenCalled()); + current.activeGenerationRef.current = 2; + current.sourceSwitchRef.current = null; + current.setNotice("new-share"); + pending.resolve(true); + await switching; + expect(current.setNotice).toHaveBeenLastCalledWith("new-share"); + expect(current.setNotice).not.toHaveBeenCalledWith("source-switch-result"); + }); + + it("uses the current committed profile when reauthentication SFU recovery finishes late", async () => { + const current = fixture(); + const pending = deferred(); + current.route.resyncAuthoritative.mockReturnValue(pending.promise); + current.reauthenticate(); + await current.change(lower); + current.route.updateProfile.mockClear(); + pending.resolve(); + await vi.waitFor(() => expect(current.route.updateProfile).toHaveBeenCalledOnce()); + expect(current.route.updateProfile).toHaveBeenCalledWith(lower); + }); +}); diff --git a/tests/native-client.test.ts b/tests/native-client.test.ts index 6aeb819..4cdad77 100644 --- a/tests/native-client.test.ts +++ b/tests/native-client.test.ts @@ -1,6 +1,7 @@ import { afterEach, describe, expect, it, vi } from "vitest"; import { NativeClient, notifyNativePresentation } from "../src/client/native/client"; +import { DEFAULT_QUALITY_SETTINGS } from "../src/shared/protocol"; import { nativeEventSchema, @@ -25,7 +26,7 @@ describe("native Client private wire", () => { }); const fetcher = vi.fn(async (url: string) => url.endsWith("/health") ? new Response(JSON.stringify({ - protocol: 8, service: "screener-client", port: 39_721, + protocol: 9, service: "screener-client", port: 39_721, instanceToken: "a".repeat(43), nativeMedia: {video: true, processAudio: false, systemAudio: true, hardwareH264: true, softwareVP8: true}, }), {status: 200}) @@ -52,7 +53,7 @@ describe("native Client private wire", () => { static readonly CLOSING = 2; static readonly CLOSED = 3; readyState = FakeWebSocket.OPEN; - protocol = `screener-client-v8.${token}`; + protocol = `screener-client-v9.${token}`; readonly close = vi.fn(() => { this.readyState = FakeWebSocket.CLOSING; }); @@ -65,14 +66,15 @@ describe("native Client private wire", () => { send(payload: string): void { const request = JSON.parse(payload) as { id: string; type: string }; - if (request.type !== "hello") return; + if (!["hello", "update-share", "ping"].includes(request.type)) return; queueMicrotask(() => { const event = new Event("message"); Object.defineProperty(event, "data", { value: JSON.stringify({ - version: 8, + version: 9, id: request.id, - type: "ready", + type: request.type === "hello" ? "ready" : request.type === "ping" ? "pong" : "request-failed", + ...(request.type === "update-share" ? { code: "operation-failed" } : {}), }), }); this.dispatchEvent(event); @@ -91,7 +93,7 @@ describe("native Client private wire", () => { }); vi.stubGlobal("fetch", vi.fn(async () => new Response(JSON.stringify({ - protocol: 8, + protocol: 9, service: "screener-client", port: 39_721, instanceToken: token, @@ -109,6 +111,11 @@ describe("native Client private wire", () => { expect(client).not.toBeNull(); const unexpected = vi.fn(); client!.onClose(unexpected); + await expect(client!.updateShare("share_123456", DEFAULT_QUALITY_SETTINGS)) + .rejects.toThrow("Screener Client request failed"); + await client!.ping(); + expect(sockets[0]!.close).not.toHaveBeenCalled(); + expect(unexpected).not.toHaveBeenCalled(); sockets[0]!.emitUnexpectedClose(); sockets[0]!.emitUnexpectedClose(); expect(unexpected).toHaveBeenCalledOnce(); @@ -125,7 +132,7 @@ describe("native Client private wire", () => { it("keeps public discovery capability-only", () => { expect( nativeHealthSchema.parse({ - protocol: 8, + protocol: 9, service: "screener-client", port: 39_721, instanceToken: "a".repeat(43), @@ -193,7 +200,7 @@ describe("native Client private wire", () => { it("fences native events by share and connection identity", () => { const event = { - version: 8, + version: 9, type: "edge-state", shareId: "share_123456", connectionId: "edge_1234567", @@ -207,7 +214,7 @@ describe("native Client private wire", () => { nativeEventSchema.safeParse({ ...event, routeRevision: 1 }).success, ).toBe(false); const path = { - version: 8, + version: 9, type: "edge-path", shareId: "share_123456", connectionId: "edge_1234567", @@ -222,7 +229,7 @@ describe("native Client private wire", () => { it("keeps an unavailable preview advisory instead of treating it as media failure", () => { const preview = { - version: 8, + version: 9, id: "request_preview", type: "source-preview", sourceKey: "display:65537", @@ -249,7 +256,7 @@ describe("native Client private wire", () => { screenAudioQuality: "very-high", }; expect(shareStartedResponseSchema.safeParse({ - version: 8, + version: 9, id: "request_start", type: "share-started", shareId: "share_123456", @@ -257,19 +264,19 @@ describe("native Client private wire", () => { codec: "h264", }).success).toBe(true); expect(shareUpdatedResponseSchema.safeParse({ - version: 8, + version: 9, id: "request_update", type: "share-updated", shareId: "share_123456", }).success).toBe(true); expect(shareSourceReplacedResponseSchema.safeParse({ - version: 8, + version: 9, id: "request_source", type: "share-source-replaced", shareId: "share_123456", }).success).toBe(true); expect(shareUpdatedResponseSchema.safeParse({ - version: 8, + version: 9, id: "request_update", type: "share-updated", shareId: "share_123456", @@ -279,7 +286,7 @@ describe("native Client private wire", () => { it("requires the actual Native codec instead of reporting Auto as media", () => { const response = { - version: 8, + version: 9, id: "request_receive", type: "receive-answer", shareId: "share_123456", @@ -290,7 +297,7 @@ describe("native Client private wire", () => { for (const codec of ["h264", "vp8"]) { expect(receiveAnswerResponseSchema.safeParse({ ...response, codec }).success).toBe(true); expect(shareStartedResponseSchema.safeParse({ - version: 8, id: "request_start", type: "share-started", + version: 9, id: "request_start", type: "share-started", shareId: response.shareId, audio: false, codec, }).success).toBe(true); } @@ -300,7 +307,7 @@ describe("native Client private wire", () => { it("accepts only internally consistent native quality evidence", () => { const event = { - version: 8, + version: 9, type: "edge-quality", shareId: "share_123456", connectionId: "edge_1234567", diff --git a/tests/native-media-bridge.test.ts b/tests/native-media-bridge.test.ts index 45154b7..e9e602a 100644 --- a/tests/native-media-bridge.test.ts +++ b/tests/native-media-bridge.test.ts @@ -2,9 +2,17 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { NativeMediaBridge, + NativeMediaBridgeError, type NativeMediaBridgeControl, } from "../src/client/native/media-bridge"; import type { NativeClientEvent } from "../src/client/native/wire"; +import { debugError, debugEvent } from "../src/client/lib/debug"; + +vi.mock("../src/client/lib/debug", () => ({ + browserDebugEnabled: true, + debugEvent: vi.fn(), + debugError: vi.fn(), +})); class FakeMediaStream { private readonly tracks: MediaStreamTrack[] = []; @@ -28,6 +36,10 @@ class FakeMediaStream { class FakePeerConnection extends EventTarget { connectionState: RTCPeerConnectionState = "new"; + iceConnectionState: RTCIceConnectionState = "new"; + iceGatheringState: RTCIceGatheringState = "new"; + readonly getReceivers = vi.fn<() => RTCRtpReceiver[]>(() => []); + readonly getStats = vi.fn(async () => new Map()); readonly addIceCandidate = vi.fn(async () => undefined); readonly setRemoteDescription = vi.fn(async () => undefined); readonly createAnswer = vi.fn(async () => ({ @@ -67,7 +79,7 @@ function fixture(options: { NativeMediaBridgeControl["prepareLocalEdge"] >(async (_shareId, connectionId) => { listener?.({ - version: 8, + version: 9, type: "edge-candidate", shareId: "share_123456", connectionId, @@ -109,6 +121,7 @@ function fixture(options: { beforeEach(() => { vi.useRealTimers(); + vi.clearAllMocks(); }); afterEach(() => { @@ -185,6 +198,57 @@ describe("native media bridge", () => { expect(current.bridge.stream.getAudioTracks()).toHaveLength(1); }); + it("reports only categorical transport and candidate counts on timeout", async () => { + vi.useFakeTimers(); + const current = fixture(); + const peer = current.peer(); + peer.connectionState = "connecting"; + peer.iceConnectionState = "checking"; + peer.iceGatheringState = "complete"; + peer.getReceivers.mockReturnValue([{ + track: { kind: "video", readyState: "live", id: "private-track" }, + transport: { state: "new" }, + } as unknown as RTCRtpReceiver]); + peer.getStats.mockResolvedValue(new Map([ + ["private-local", { type: "local-candidate", candidateType: "host", protocol: "tcp", address: "192.0.2.1", port: 54321 }], + ["private-remote", { type: "remote-candidate", candidateType: "host", protocol: "udp", address: "192.0.2.2", port: 54322 }], + ])); + const starting = current.bridge.start(); + const rejected = expect(starting).rejects.toBeInstanceOf(NativeMediaBridgeError); + await vi.advanceTimersByTimeAsync(8_000); + await rejected; + + expect(debugEvent).toHaveBeenCalledWith("native-bridge", "failed", { type: "timeout", state: "starting" }); + expect(debugEvent).toHaveBeenCalledWith("native-bridge", "ice-state", { state: "checking" }); + expect(debugEvent).toHaveBeenCalledWith("native-bridge", "ice-gathering-state", { state: "complete" }); + expect(debugEvent).toHaveBeenCalledWith("native-bridge", "dtls-state", { state: "new" }); + expect(debugEvent).toHaveBeenCalledWith("native-bridge", "receiver-count", { type: "video", state: "live", count: 1 }); + expect(debugEvent).toHaveBeenCalledWith("native-bridge", "candidate-count", { type: "local-host", state: "udp", count: 0 }); + expect(debugEvent).toHaveBeenCalledWith("native-bridge", "candidate-count", { type: "remote-host", state: "udp", count: 1 }); + const report = JSON.stringify(vi.mocked(debugEvent).mock.calls); + for (const privateValue of ["private-track", "private-local", "private-remote", "192.0.2.", "54321", "54322"]) { + expect(report).not.toContain(privateValue); + } + expect(peer.getStats).toHaveBeenCalledOnce(); + expect(peer.close).toHaveBeenCalledOnce(); + }); + + it.each(["failed", "pending"])("does not delay timeout cleanup for %s diagnostics", async (state) => { + vi.useFakeTimers(); + const current = fixture(); + current.peer().getStats.mockImplementation(() => state === "failed" + ? Promise.reject(new Error("private stats failure")) + : new Promise(() => undefined)); + const rejected = expect(current.bridge.start()).rejects.toBeInstanceOf(NativeMediaBridgeError); + await vi.advanceTimersByTimeAsync(8_000); + await rejected; + expect(current.peer().close).toHaveBeenCalledOnce(); + expect(current.control.closeEdge).toHaveBeenCalledOnce(); + if (state === "failed") expect(debugError).toHaveBeenCalledWith( + "native-bridge", "diagnostics-failed", expect.any(Error), + ); + }); + it("reports an active native failure once and retires its edge", async () => { const current = fixture(); const starting = current.bridge.start(); @@ -199,14 +263,14 @@ describe("native media bridge", () => { await starting; current.emit({ - version: 8, + version: 9, type: "edge-state", shareId: "share_123456", connectionId: current.bridge.connectionId, state: "failed", }); current.emit({ - version: 8, + version: 9, type: "edge-state", shareId: "share_123456", connectionId: current.bridge.connectionId, diff --git a/tests/native-media-ingress.test.ts b/tests/native-media-ingress.test.ts index 654d9d9..ea9ecfe 100644 --- a/tests/native-media-ingress.test.ts +++ b/tests/native-media-ingress.test.ts @@ -44,7 +44,7 @@ function fixture(pending = false) { onEvent: (next) => { listener = next; return () => { listener = null; }; }, }; const onFailed = vi.fn(); - const ingress = new NativeMediaIngress("share_123456", control, onFailed); + const ingress = new NativeMediaIngress("share_123456", control, onFailed, () => DEFAULT_QUALITY_SETTINGS); const stream = { getAudioTracks: () => [] } as unknown as MediaStream; return { ingress, control, stream, onFailed, answer: () => resolveAnswer(answer), emit: (event: NativeClientEvent) => listener?.(event) }; } diff --git a/tests/native-sender-edge.test.ts b/tests/native-sender-edge.test.ts index 40854aa..7654bf9 100644 --- a/tests/native-sender-edge.test.ts +++ b/tests/native-sender-edge.test.ts @@ -14,6 +14,7 @@ function fixture(natPrediction = false) { sdp: "v=0\r\n", })); const control: NativeEdgeControl = { + updateShare: vi.fn(async () => undefined), prepareEdge, acceptSignal: vi.fn(async () => undefined), closeEdge: vi.fn(async () => undefined), @@ -64,7 +65,7 @@ describe("native sender edge adapter", () => { const prepare = vi.mocked(current.control.prepareEdge); prepare.mockImplementationOnce(async () => { current.emit({ - version: 8, + version: 9, type: "edge-candidate", shareId: "share_1234567", connectionId: "edge_12345678", @@ -85,7 +86,7 @@ describe("native sender edge adapter", () => { const current = fixture(); expect(await current.edge.start()).toBe(true); current.emit({ - version: 8, + version: 9, type: "edge-state", shareId: "share_1234567", connectionId: "edge_12345678", @@ -113,7 +114,7 @@ describe("native sender edge adapter", () => { .toHaveLength(3); for (const [index, port] of [40_000, 40_003, 40_006].entries()) { current.emit({ - version: 8, + version: 9, type: "edge-candidate", shareId: "share_1234567", connectionId: "edge_12345678", diff --git a/tests/native-sender-peer.test.ts b/tests/native-sender-peer.test.ts index 228c4ca..439a122 100644 --- a/tests/native-sender-peer.test.ts +++ b/tests/native-sender-peer.test.ts @@ -1,4 +1,5 @@ import { describe, expect, it, vi } from "vitest"; +import { DEFAULT_QUALITY_SETTINGS, type QualitySettings } from "../src/shared/protocol"; import { invalidateSenderQualityEvidence, @@ -17,6 +18,7 @@ describe("native Host peer quality", () => { invalidateSenderQualityEvidence(); let listener: (event: NativeClientEvent) => void = () => undefined; const control: NativeEdgeControl = { + updateShare: vi.fn(async () => undefined), prepareEdge: vi.fn(async () => ({ type: "offer", sdp: "v=0\r\n", @@ -45,6 +47,7 @@ describe("native Host peer quality", () => { codec, ); expect(await peer.start()).toBe(true); + expect(control.updateShare).not.toHaveBeenCalled(); expect(shouldUseBrowserQualityCandidate(peer, { childPeerId: "viewer_123456", connectionId: "candidate_123456", @@ -64,7 +67,7 @@ describe("native Host peer quality", () => { qualityProbe: true, })).toBe(false); listener({ - version: 8, + version: 9, type: "edge-state", shareId: "share_1234567", connectionId: "edge_12345678", @@ -72,7 +75,7 @@ describe("native Host peer quality", () => { }); const quality = { - version: 8 as const, + version: 9 as const, type: "edge-quality" as const, shareId: "share_1234567", connectionId: "edge_12345678", @@ -110,4 +113,29 @@ describe("native Host peer quality", () => { }); peer.dispose(); }); + + it("applies the latest relay ceiling before preparing a child", async () => { + let profile: QualitySettings = DEFAULT_QUALITY_SETTINGS; + const control: NativeEdgeControl = { + updateShare: vi.fn(async () => undefined), + prepareEdge: vi.fn(async () => ({ type: "offer" as const, sdp: "v=0\r\n" })), + acceptSignal: vi.fn(async () => undefined), + closeEdge: vi.fn(async () => undefined), + onEvent: () => () => undefined, + }; + const peer = new NativeSenderPeer( + "viewer_123456", "edge_12345678", "share_1234567", + { iceServers: [] }, false, control, + { sendSignal: () => true, onUpdate: () => undefined }, "h264", + { connectionId: "upstream_123456", getProfile: () => profile }, + ); + profile = { ...DEFAULT_QUALITY_SETTINGS, maxFramerate: 60 }; + expect(await peer.start()).toBe(true); + expect(control.updateShare).toHaveBeenCalledWith("share_1234567", profile); + expect(vi.mocked(control.updateShare).mock.invocationCallOrder[0]) + .toBeLessThan(vi.mocked(control.prepareEdge).mock.invocationCallOrder[0]!); + expect(await peer.updateCaptureProfile(DEFAULT_QUALITY_SETTINGS)).toBe(true); + expect(control.updateShare).toHaveBeenLastCalledWith("share_1234567", DEFAULT_QUALITY_SETTINGS); + peer.dispose(); + }); }); diff --git a/tests/native-sfu-publisher.test.ts b/tests/native-sfu-publisher.test.ts new file mode 100644 index 0000000..4e73792 --- /dev/null +++ b/tests/native-sfu-publisher.test.ts @@ -0,0 +1,148 @@ +import { describe, expect, it, vi } from "vitest"; +import { NativeSfuPublisher } from "../src/client/native/native-sfu-publisher"; +import type { NativeClientEvent } from "../src/client/native/wire"; +import type { SfuSignalMessage } from "../src/shared/protocol"; + +describe("Native SFU publication", () => { + it("forwards existing Native media with exact identity and bounded signaling replay", async () => { + const config = { + revision: 3, + publicationGeneration: "publication_generation", + connectionId: "publication_connection", + }; + let listener: ((event: NativeClientEvent) => void) | null = null; + const emit = (event: NativeClientEvent) => listener?.(event); + const media = { + codec: "vp8" as const, + layers: [ + { rid: "q", width: 320, height: 180, bitrate: 300_000 }, + { rid: "h", width: 640, height: 360, bitrate: 1_200_000 }, + ], + audio: true, + audioBitrate: 64_000, + }; + const control = { + preparePublication: vi.fn(async () => { + emit({ + version: 9, + type: "publication-candidate", + shareId: "native_share", + ...config, + candidate: { + candidate: "candidate:1 1 udp 1 127.0.0.1 1234 typ host", + }, + }); + return { + description: { type: "offer" as const, sdp: "v=0\r\n" }, + media, + }; + }), + publicationMedia: vi.fn(async () => media), + acceptPublicationSignal: vi.fn(async () => undefined), + closePublication: vi.fn(async () => undefined), + onEvent: vi.fn((next: (event: NativeClientEvent) => void) => { + listener = next; + return () => { + listener = null; + }; + }), + }; + const sent: SfuSignalMessage[] = []; + let connected = false; + const onDisconnected = vi.fn(); + const onStats = vi.fn(); + const publisher = new NativeSfuPublisher( + control, + "native_share", + { iceServers: [] }, + { + send: (message) => { + if (!connected) return false; + sent.push(message); + return true; + }, + onDisconnected, + onStats, + }, + ); + expect(await publisher.connect(config)).toBe(true); + expect(await publisher.activate()).toBe(true); + expect(sent).toEqual([]); + connected = true; + const routeSlot = { ...config, revision: 4, publisher }; + publisher.updateConfig(routeSlot); + expect(sent.map(({ kind }) => kind)).toEqual(["description", "candidate"]); + expect(sent[0]).toMatchObject({ revision: 4, media }); + expect(sent[0]).not.toHaveProperty("publisher"); + expect(() => JSON.stringify(sent)).not.toThrow(); + const demand: SfuSignalMessage = { + type: "sfu-signal", + ...config, + kind: "layers", + activeCount: 1, + }; + await publisher.acceptSignal({ + ...demand, + publicationGeneration: "stale_generation", + }); + expect(control.acceptPublicationSignal).not.toHaveBeenCalled(); + await publisher.acceptSignal(demand); + expect(control.acceptPublicationSignal).toHaveBeenCalledWith( + "native_share", + demand, + ); + expect(await publisher.updateProfile()).toBe(true); + expect(sent.at(-1)).toMatchObject({ kind: "media", media }); + emit({ + version: 9, + type: "publication-quality", + shareId: "native_share", + ...config, + sampleTimestampMs: 12_000, + sampleWindowMs: 2_000, + rtpStatsId: "native_rtp_stats", + trackIdentifier: "screen", + codec: "vp8", + videoEncodingCount: 2, + activeVideoEncodingCount: 2, + rid: "h", + intervalFramesSent: 30, + framesPerSecond: 15, + width: 640, + height: 360, + bitrateKbps: 1200, + audioBitrateKbps: 64, + availableOutgoingKbps: 1600, + state: "healthy", + reason: "none", + }); + expect(onStats).toHaveBeenCalledWith( + expect.objectContaining({ + framesPerSecond: 15, + activeVideoEncodingCount: 2, + bitrateKbps: 1200, + intervalFramesSent: 30, + intervalFramesEncoded: null, + intervalEncodeMs: null, + captureFramesPerSecond: null, + frameWidth: 640, + frameHeight: 360, + }), + ); + emit({ + version: 9, + type: "publication-state", + shareId: "native_share", + ...config, + state: "failed", + }); + expect(onDisconnected).toHaveBeenCalledOnce(); + expect(control.closePublication).toHaveBeenCalledWith( + "native_share", + config.publicationGeneration, + config.connectionId, + ); + await publisher.disconnect(); + expect(control.closePublication).toHaveBeenCalledOnce(); + }); +}); diff --git a/tests/native-viewer-peer.test.ts b/tests/native-viewer-peer.test.ts index 80583df..6576743 100644 --- a/tests/native-viewer-peer.test.ts +++ b/tests/native-viewer-peer.test.ts @@ -1,9 +1,18 @@ -import { expect, it } from "vitest"; +import { afterEach, expect, it, vi } from "vitest"; +import type { SignalPayload } from "../src/shared/protocol"; +import type { NativeClient } from "../src/client/native/client"; +import { NativeMediaBridge } from "../src/client/native/media-bridge"; +import { ViewerPeer } from "../src/client/webrtc/viewer-peer"; import { NativeCapableViewerPeer, offerHasNativeVideoCodec, } from "../src/client/native/native-viewer-peer"; +afterEach(() => { + vi.restoreAllMocks(); + vi.unstubAllGlobals(); +}); + it("admits one active sending H264 or VP8 video section, not unrelated SDP codec text", () => { const video = (codec: string, direction = "sendonly", port = 9, payload = 96) => [ `m=video ${port} UDP/TLS/RTP/SAVPF ${payload}`, @@ -36,7 +45,8 @@ it("does not create a browser backend after deferred native discovery is dispose onUpdate: () => undefined, }, {}, - nativeClient, + () => nativeClient, + () => undefined, "session", 2, ); @@ -59,3 +69,70 @@ it("does not create a browser backend after deferred native discovery is dispose await accepted; expect(peer.getConnectionIdentity()).toBeNull(); }); + +it.each(["route", "viewer"] as const)("keeps %s bridge failure unavailable across replacement peers", async (recoveryOwner) => { + vi.stubGlobal("window", globalThis); + vi.stubGlobal("MediaStream", class { getTracks() { return []; } }); + vi.stubGlobal("RTCPeerConnection", class { close() {} }); + const browserSignal = vi.spyOn(ViewerPeer.prototype, "acceptSignal").mockResolvedValue(); + const bridgeStart = vi.spyOn(NativeMediaBridge.prototype, "start").mockRejectedValue(new Error("bridge failed")); + const receiveOffer = vi.fn(async () => ({ + answer: { type: "answer", sdp: "v=0\r\n" }, audio: false, codec: "vp8", + })); + const client = { + receiveOffer, + onEvent: () => () => undefined, + onClose: () => () => undefined, + closeReceiver: vi.fn(async () => undefined), + closeEdge: vi.fn(async () => undefined), + } as unknown as NativeClient; + let available = true; + const acquire = vi.fn(async () => available ? client : null); + const unavailable = vi.fn(() => { available = false; }); + const recover = vi.fn(() => { + expect(available).toBe(false); + return true; + }); + const create = () => new NativeCapableViewerPeer( + { iceServers: [] }, + { + sendSignal: () => true, + sendRestartRequest: recover, + onStream: () => undefined, + onUpdate: () => undefined, + onRecoveryExhausted: recover, + }, + { recoveryOwner }, + acquire, + unavailable, + "session", + 2, + ); + const offer = (connectionId: string): SignalPayload => ({ + kind: "description", connectionId, + description: { type: "offer", sdp: "v=0\r\nm=video 9 UDP/TLS/RTP/SAVPF 96\r\na=sendonly\r\na=rtpmap:96 VP8/90000\r\n" }, + }); + const first = create(); + const queued = create(); + try { + await first.acceptSignal("parent", offer("first")); + await vi.waitFor(() => expect(unavailable).toHaveBeenCalledOnce()); + expect(recover).toHaveBeenCalledOnce(); + first.dispose(); + await queued.acceptSignal("parent", offer("queued")); + expect(browserSignal).toHaveBeenCalledOnce(); + const replacement = create(); + try { + await replacement.acceptSignal("parent", offer("replacement")); + expect(browserSignal).toHaveBeenCalledTimes(2); + expect(receiveOffer).toHaveBeenCalledOnce(); + expect(bridgeStart).toHaveBeenCalledOnce(); + expect(acquire).toHaveBeenCalledTimes(3); + } finally { + replacement.dispose(); + } + } finally { + first.dispose(); + queued.dispose(); + } +}); diff --git a/tests/protocol-fixture.test.ts b/tests/protocol-fixture.test.ts index c4e9ea0..063d5b9 100644 --- a/tests/protocol-fixture.test.ts +++ b/tests/protocol-fixture.test.ts @@ -17,7 +17,6 @@ import { MAX_MEDIA_ROUTE_REVISION, MAX_NAT_PREDICTION_AUXILIARY_STUN_URLS, MAX_PARTICIPANTS_PER_ROOM_LIMIT, - MAX_SFU_TOKEN_LENGTH, MAX_SIGNAL_BYTES, MAX_VIEWER_PASSWORD_LENGTH, MAX_VIEWER_QUALITY_EVIDENCE_BYTES, @@ -77,7 +76,6 @@ describe("shared wire fixture", () => { maxSignalBytes: MAX_SIGNAL_BYTES, roomCodeLength: ROOM_CODE_LENGTH, maxMediaRouteRevision: MAX_MEDIA_ROUTE_REVISION, - maxSfuTokenLength: MAX_SFU_TOKEN_LENGTH, maxIceServerUrls: MAX_ICE_SERVER_URLS, maxNatPredictionAuxiliaryStunUrls: MAX_NAT_PREDICTION_AUXILIARY_STUN_URLS, maxViewerQualityEvidenceBytes: MAX_VIEWER_QUALITY_EVIDENCE_BYTES, @@ -134,6 +132,7 @@ describe("shared wire fixture", () => { "client:set-quality-settings", "client:set-sharing-paused", "client:sfu-publisher-quality-evidence", + "client:sfu-signal", "client:signal", "client:signaling-challenge", "client:stop-sharing", @@ -176,6 +175,7 @@ describe("shared wire fixture", () => { "route-status", "route-update", "sfu-config", + "sfu-signal", "sharing-stopped", "signal", "signaling-challenge-response", diff --git a/tests/protocol.test.ts b/tests/protocol.test.ts index 0885fac..76eb7ab 100644 --- a/tests/protocol.test.ts +++ b/tests/protocol.test.ts @@ -6,7 +6,6 @@ import { DEFAULT_VIEWER_DISPLAY_NAME, MAX_DISPLAY_NAME_CODE_POINTS, MAX_MEDIA_ROUTE_REVISION, - MAX_SFU_TOKEN_LENGTH, MAX_VIEWER_QUALITY_EVIDENCE_BYTES, MAX_VIEWER_PASSWORD_LENGTH, MAX_VIEWERS_PER_ROOM_LIMIT, @@ -88,7 +87,7 @@ const qualityEvidence = { describe("client signaling protocol", () => { it("uses the current strict signaling generation", () => { - expect(SIGNALING_PROTOCOL).toBe("screener-v21"); + expect(SIGNALING_PROTOCOL).toBe("screener-v22"); }); it("keeps signaling challenges strict and sequence-only", () => { @@ -1426,12 +1425,6 @@ describe("server signaling protocol", () => { }; expect(serverMessageSchema.safeParse(peerAssisted).success).toBe(true); - expect( - serverMessageSchema.safeParse({ - ...peerAssisted, - sfuStandbyUrl: "wss://sfu.example.com", - }).success, - ).toBe(true); const incomplete = { ...authenticatedMessage(8), } as Record; @@ -1462,12 +1455,6 @@ describe("server signaling protocol", () => { } as Record; delete missingMode.mediaMode; expect(serverMessageSchema.safeParse(missingMode).success).toBe(false); - expect( - serverMessageSchema.safeParse({ - ...peerAssisted, - sfuStandbyUrl: "https://sfu.example.com", - }).success, - ).toBe(false); expect( serverMessageSchema.safeParse({ type: "quality-settings", @@ -1564,8 +1551,8 @@ describe("server signaling protocol", () => { serverMessageSchema.safeParse({ type: "sfu-config", revision: 9, - url: "wss://sfu.example.com", - token: "header.payload.signature", + publicationGeneration: "publication_12345678", + connectionId: "connection_12345678", }).success, ).toBe(true); @@ -1599,8 +1586,8 @@ describe("server signaling protocol", () => { serverMessageSchema.safeParse({ type: "sfu-config", revision: 9, - url: "wss://sfu.example.com", - token: "x".repeat(MAX_SFU_TOKEN_LENGTH + 1), + publicationGeneration: "publication_12345678", + connectionId: "", }).success, ).toBe(false); expect( diff --git a/tests/route-transition.test.ts b/tests/route-transition.test.ts index cd914a2..139abc4 100644 --- a/tests/route-transition.test.ts +++ b/tests/route-transition.test.ts @@ -51,17 +51,19 @@ const hostAssignment = ( sfuPublicationGeneration: publicationGeneration, }); -function sfuConfig(revision: number) { +function sfuConfig(revision: number, publicationGeneration = "publication_generation_12345678") { return { type: "sfu-config" as const, revision, - url: "wss://sfu.example.test", - token: `token-${revision}`, + publicationGeneration, + connectionId: `sfu_connection_${revision}_12345678`, }; } function createFakePublisher(log: string[], label: string) { return { + updateConfig: vi.fn(), + acceptSignal: vi.fn(async () => undefined), connect: vi.fn(async (_config: unknown) => { log.push(`${label}:connect`); return true; @@ -102,6 +104,9 @@ const candidate = ( function createFakeSubscriber(events: SubscriberEvents, log: string[], label: string) { return { + updateConfig: vi.fn(), + acceptSignal: vi.fn(async () => undefined), + reconnect: vi.fn(() => true), events, connect: vi.fn(async () => { log.push(`${label}:connect`); @@ -388,7 +393,7 @@ describe("minimal route transition contracts", () => { phase: "active", assignment: hostAssignment("publication-a"), }); - await route.acceptConfig(sfuConfig(1)); + await route.acceptConfig(sfuConfig(1, "publication-a")); expect(publishers[0]?.activate).toHaveBeenCalledOnce(); expect(publishers[0]?.activate).toHaveBeenCalledWith( expect.anything(), @@ -402,7 +407,7 @@ describe("minimal route transition contracts", () => { assignment: hostAssignment("publication-b"), candidate: candidate(2, "viewer_12345678", "sfu"), }); - await route.acceptConfig(sfuConfig(2)); + await route.acceptConfig(sfuConfig(2, "publication-b")); expect(publishers[1]?.activate).toHaveBeenCalledOnce(); expect(publishers[0]?.deactivate).not.toHaveBeenCalled(); expect(messages.filter((message) => message.type === "route-ready")).toEqual([]); @@ -432,7 +437,7 @@ describe("minimal route transition contracts", () => { phase: "active", assignment: hostAssignment("publication", ["committed-child"]), }); - await route.acceptConfig(sfuConfig(1)); + await route.acceptConfig(sfuConfig(1, "publication")); reconciledChildren.length = 0; route.accept({ @@ -451,6 +456,35 @@ describe("minimal route transition contracts", () => { expect(reconciledChildren).toEqual([["candidate-child"]]); }); + it("replaces a Host physical connection within one publication and fences stale signaling", async () => { + const publishers: ReturnType[] = []; + const route = new HostSfuRoute({ + getStream: () => ({}) as MediaStream, + getProfile: () => QUALITY_PROFILES["720p30"], + getVideoCodec: () => "vp8", + reconcileChildren: () => undefined, + send: () => true, + createPublisher: () => { + const publisher = createFakePublisher([], "publisher"); + publishers.push(publisher); + return publisher; + }, + }); + await route.acceptAndWait({ revision: 1, phase: "active", assignment: hostAssignment("publication") }); + const first = sfuConfig(1, "publication"); + await route.acceptConfig(first); + const replacement = { ...first, connectionId: "replacement_connection" }; + await route.acceptConfig(replacement); + expect(publishers).toHaveLength(2); + expect(publishers[0]!.disconnect).toHaveBeenCalledOnce(); + expect(publishers[1]!.disconnect).not.toHaveBeenCalled(); + await route.acceptSignal({ ...first, type: "sfu-signal", kind: "candidate", candidate: { candidate: "old" } }); + expect(publishers[0]!.acceptSignal).not.toHaveBeenCalled(); + await route.acceptSignal({ ...replacement, type: "sfu-signal", kind: "candidate", candidate: { candidate: "current" } }); + expect(publishers[1]!.acceptSignal).toHaveBeenCalledOnce(); + await route.disconnect(); + }); + it("reports and retires an exact active SFU source replacement failure", async () => { const messages: ClientMessage[] = []; const publishers: ReturnType[] = []; @@ -475,7 +509,7 @@ describe("minimal route transition contracts", () => { phase: "active", assignment: hostAssignment("publication-active"), }); - await route.acceptConfig(sfuConfig(1)); + await route.acceptConfig(sfuConfig(1, "publication-active")); publishers[0]!.replaceStream.mockResolvedValue(false); await expect(route.replaceStream({} as MediaStream)).resolves.toBe(false); @@ -510,14 +544,14 @@ describe("minimal route transition contracts", () => { phase: "active", assignment: hostAssignment("publication-active"), }); - await route.acceptConfig(sfuConfig(1)); + await route.acceptConfig(sfuConfig(1, "publication-active")); route.accept({ revision: 2, phase: "prepare", assignment: hostAssignment("publication-pending"), candidate: candidate(2, "viewer_12345678", "sfu"), }); - await route.acceptConfig(sfuConfig(2)); + await route.acceptConfig(sfuConfig(2, "publication-pending")); publishers[1]!.replaceStream.mockResolvedValue(false); await expect(route.replaceStream({} as MediaStream)).resolves.toBe(true); @@ -554,14 +588,14 @@ describe("minimal route transition contracts", () => { phase: "active", assignment: hostAssignment("publication-old"), }); - await route.acceptConfig(sfuConfig(1)); + await route.acceptConfig(sfuConfig(1, "publication-old")); route.accept({ revision: 2, phase: "prepare", assignment: hostAssignment("publication-new"), candidate: candidate(2, "viewer_12345678", "sfu"), }); - await route.acceptConfig(sfuConfig(2)); + await route.acceptConfig(sfuConfig(2, "publication-new")); let resolveOldReplacement!: (replaced: boolean) => void; publishers[0]!.replaceStream.mockImplementation( @@ -615,7 +649,7 @@ describe("minimal route transition contracts", () => { phase: "active", assignment: hostAssignment("publication-old"), }); - await route.acceptConfig(sfuConfig(1)); + await route.acceptConfig(sfuConfig(1, "publication-old")); route.setPaused(true); expect(publishers[0]?.setPaused).toHaveBeenCalledWith(true); @@ -673,12 +707,12 @@ describe("minimal route transition contracts", () => { phase: "active", assignment: hostAssignment("publication-paused"), }); - await route.acceptConfig(sfuConfig(1)); + await route.acceptConfig(sfuConfig(1, "publication-paused")); route.setPaused(true); disconnects[0]!(); expect(messages).toContainEqual({ type: "refresh-sfu", revision: 1 }); - await route.acceptConfig({ ...sfuConfig(1), token: "recovery-token" }); + await route.acceptConfig({ ...sfuConfig(1, "publication-paused"), connectionId: "recovery_connection" }); expect(publishers).toHaveLength(2); expect(publishers[1]!.setPaused).toHaveBeenCalledWith(true); @@ -719,7 +753,7 @@ describe("minimal route transition contracts", () => { phase: "active", assignment: hostAssignment("publication-old"), }); - await route.acceptConfig(sfuConfig(1)); + await route.acceptConfig(sfuConfig(1, "publication-old")); route.accept({ revision: 2, @@ -732,7 +766,7 @@ describe("minimal route transition contracts", () => { "candidate_2_12345678", ), }); - const freshPublication = route.acceptConfig(sfuConfig(2)); + const freshPublication = route.acceptConfig(sfuConfig(2, "publication-fresh")); await vi.waitFor(() => expect(publishers[1]?.activate).toHaveBeenCalledOnce(), ); @@ -786,14 +820,14 @@ describe("minimal route transition contracts", () => { phase: "active", assignment: hostAssignment("publication-old"), }); - await route.acceptConfig(sfuConfig(1)); + await route.acceptConfig(sfuConfig(1, "publication-old")); route.accept({ revision: 2, phase: "prepare", assignment: hostAssignment("publication-new"), candidate: candidate(2, "viewer_12345678", "sfu"), }); - const stale = route.acceptConfig(sfuConfig(2)); + const stale = route.acceptConfig(sfuConfig(2, "publication-new")); await vi.waitFor(() => expect(publishers[1]?.connect).toHaveBeenCalledOnce()); await route.resyncAuthoritative({ @@ -807,7 +841,7 @@ describe("minimal route transition contracts", () => { assignment: hostAssignment("publication-new"), candidate: candidate(2, "viewer_12345678", "sfu"), }); - const current = route.acceptConfig(sfuConfig(2)); + const current = route.acceptConfig(sfuConfig(2, "publication-new")); await vi.waitFor(() => expect(publishers).toHaveLength(3)); releaseOldConnect(); await Promise.all([stale, current]); @@ -855,14 +889,14 @@ describe("minimal route transition contracts", () => { phase: "active", assignment: viewerSfuAssignment([], "publication-old"), }); - await route.acceptConfig(sfuConfig(1)); + await route.acceptConfig(sfuConfig(1, "publication-old")); route.accept({ revision: 2, phase: "prepare", assignment: viewerSfuAssignment([], "publication-new"), candidate: candidate(2, "viewer_12345678", "sfu"), }); - const stale = route.acceptConfig(sfuConfig(2)); + const stale = route.acceptConfig(sfuConfig(2, "publication-new")); await vi.waitFor(() => expect(subscribers[1]?.connect).toHaveBeenCalledOnce()); await route.resyncAuthoritative( @@ -879,7 +913,7 @@ describe("minimal route transition contracts", () => { assignment: viewerSfuAssignment([], "publication-new"), candidate: candidate(2, "viewer_12345678", "sfu"), }); - const current = route.acceptConfig(sfuConfig(2)); + const current = route.acceptConfig(sfuConfig(2, "publication-new")); await vi.waitFor(() => expect(subscribers).toHaveLength(3)); releaseOldConnect(); await Promise.all([stale, current]); @@ -950,7 +984,7 @@ describe("minimal route transition contracts", () => { assignment: viewerSfuAssignment([], "publication-generation-2"), candidate: candidate(2, "viewer_12345678", "sfu"), }); - await route.acceptConfig(sfuConfig(2)); + await route.acceptConfig(sfuConfig(2, "publication-generation-2")); const pendingStream = {} as MediaStream; subscribers[1]?.events.onStream(pendingStream); subscribers[1]?.events.onStats?.({ @@ -1158,25 +1192,15 @@ describe("minimal route transition contracts", () => { await vi.waitFor(() => expect(streams).toEqual([firstStream])); expect(route.reconnectActive()).toBe(true); - expect(route.reconnectActive()).toBe(false); - await vi.waitFor(() => - expect(messages.at(-1)).toEqual({ type: "refresh-sfu", revision: 7 }), - ); + expect(subscribers[0]?.reconnect).toHaveBeenCalledOnce(); expect(resetMedia).not.toHaveBeenCalled(); expect(subscribers[0]?.deactivate).not.toHaveBeenCalled(); expect(subscribers[0]?.disconnect).not.toHaveBeenCalled(); - await route.acceptConfig({ ...sfuConfig(7), token: "fresh-token" }); - const nextStream = {} as MediaStream; - subscribers[1]?.events.onStream(nextStream); - subscribers[1]?.events.onFirstDecodedFrame(); - await vi.waitFor(() => expect(streams).toEqual([firstStream, nextStream])); - expect(subscribers[1]?.connect).toHaveBeenCalledWith({ - url: "wss://sfu.example.test", - token: "fresh-token", - }); - expect(subscribers[0]?.deactivate).toHaveBeenCalledOnce(); - expect(subscribers[0]?.disconnect).toHaveBeenCalledOnce(); + await route.acceptConfig(sfuConfig(7)); + expect(subscribers).toHaveLength(1); + expect(streams).toEqual([firstStream]); + expect(subscribers[0]?.disconnect).not.toHaveBeenCalled(); await route.disconnect(); }); @@ -1217,7 +1241,7 @@ describe("minimal route transition contracts", () => { await route.disconnect(); }); - it("retargets manual SFU recovery after an unrelated room revision", async () => { + it("retains the physical SFU subscriber when a room revision advances during reconnect", async () => { const messages: ClientMessage[] = []; const subscribers: ReturnType[] = []; const streams: MediaStream[] = []; @@ -1246,19 +1270,15 @@ describe("minimal route transition contracts", () => { expect(route.reconnectActive()).toBe(true); route.accept({ revision: 8, phase: "active", assignment }); - await vi.waitFor(() => - expect( - messages.filter((message) => message.type === "refresh-sfu"), - ).toEqual([ - { type: "refresh-sfu", revision: 7 }, - { type: "refresh-sfu", revision: 8 }, - ]), - ); + await vi.waitFor(() => expect(subscribers[0]?.updateConfig).toHaveBeenLastCalledWith( + expect.objectContaining({ revision: 8 }), + )); + expect(messages.filter((message) => message.type === "refresh-sfu")).toEqual([]); expect(subscribers[0]?.disconnect).not.toHaveBeenCalled(); await route.disconnect(); }); - it("retargets manual SFU recovery through an unrelated prepare", async () => { + it("keeps SFU reconnect on the physical subscriber through an unrelated prepare", async () => { const messages: ClientMessage[] = []; const streams: MediaStream[] = []; const subscribers: ReturnType[] = []; @@ -1292,14 +1312,12 @@ describe("minimal route transition contracts", () => { }); route.accept({ revision: 8, phase: "active", assignment }); - await vi.waitFor(() => - expect( - messages.filter((message) => message.type === "refresh-sfu"), - ).toEqual([ - { type: "refresh-sfu", revision: 7 }, - { type: "refresh-sfu", revision: 8 }, - ]), - ); + await vi.waitFor(() => expect(subscribers[0]?.updateConfig).toHaveBeenLastCalledWith( + expect.objectContaining({ revision: 8 }), + )); + expect(subscribers[0]?.reconnect).toHaveBeenCalledOnce(); + expect(subscribers[0]?.disconnect).not.toHaveBeenCalled(); + expect(messages.filter((message) => message.type === "refresh-sfu").every((message) => message.revision === 7)).toBe(true); }); it("does not spend SFU recovery until refresh signaling is sent", async () => { @@ -1380,7 +1398,7 @@ describe("minimal route transition contracts", () => { expect(route.reconnectActive()).toBe(true); const recovery = route.acceptConfig({ ...sfuConfig(7), - token: "recovery-token", + connectionId: "recovery_connection", }); await vi.waitFor(() => expect(subscribers).toHaveLength(2)); route.accept({ revision: 8, phase: "active", assignment }); @@ -1468,7 +1486,7 @@ describe("minimal route transition contracts", () => { subscribers[0]!.events.onDisconnected(); expect(messages).toContainEqual({ type: "refresh-sfu", revision: 7 }); - await route.acceptConfig({ ...sfuConfig(7), token: "recovery-token" }); + await route.acceptConfig({ ...sfuConfig(7), connectionId: "recovery_connection" }); expect(subscribers).toHaveLength(2); expect(subscribers[1]!.armDecodedFrameProof).not.toHaveBeenCalled(); @@ -1510,7 +1528,7 @@ describe("minimal route transition contracts", () => { phase: "active", assignment: viewerSfuAssignment([], "publication-old"), }); - await route.acceptConfig(sfuConfig(1)); + await route.acceptConfig(sfuConfig(1, "publication-old")); const oldStream = {} as MediaStream; subscribers[0]?.events.onStream(oldStream); subscribers[0]?.events.onFirstDecodedFrame(); diff --git a/tests/sender-quality-evidence.test.ts b/tests/sender-quality-evidence.test.ts index da79ac0..541f95f 100644 --- a/tests/sender-quality-evidence.test.ts +++ b/tests/sender-quality-evidence.test.ts @@ -6,6 +6,7 @@ import { sfuPublisherQualityEvidenceFromMetrics, } from "../src/client/media/sender-quality-evidence.ts"; import type { PeerSnapshot } from "../src/client/types.ts"; +import { EMPTY_METRICS } from "../src/client/types.ts"; import { collectConnectionMetricsFromReport, collectNativeSenderQualityFromReport, @@ -37,6 +38,27 @@ function report( } describe("native sender quality evidence", () => { + it("uses actual Native publication sent frames without inventing encoder work", () => { + invalidateSenderQualityEvidence(); + const metrics = { + ...EMPTY_METRICS, + sampleTimestampMs: 12_000, + sampleWindowMs: 2_000, + intervalFramesSent: 30, + framesPerSecond: 15, + nativeEdgeQualityState: "healthy" as const, + qualityLimitationReason: "none", + }; + sfuPublisherQualityEvidenceFromMetrics(metrics, 3, "native_publication"); + expect( + sfuPublisherQualityEvidenceFromMetrics( + { ...metrics, sampleTimestampMs: 14_000 }, + 3, + "native_publication", + ), + ).toMatchObject({ state: "healthy", diagnostics: { framesPerSecond: 15 } }); + expect(metrics.intervalFramesEncoded).toBeNull(); + }); it("conservatively combines every active SFU publication encoding", () => { const accumulator = createNativeSenderQualityAccumulator(); const multi = ( @@ -67,8 +89,7 @@ describe("native sender quality evidence", () => { qualityLimitationReason: reason, qualityLimitationDurations: { none: reason === "none" ? timestamp / 1_000 : 3, - bandwidth: - reason === "bandwidth" ? timestamp / 1_000 - 3 : 0, + bandwidth: reason === "bandwidth" ? timestamp / 1_000 - 3 : 0, cpu: 0, other: 0, }, diff --git a/tests/sfu-client.test.ts b/tests/sfu-client.test.ts index aaa2603..85c5803 100644 --- a/tests/sfu-client.test.ts +++ b/tests/sfu-client.test.ts @@ -1,2515 +1,551 @@ import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; - import { QUALITY_PROFILES } from "../src/client/media/quality.ts"; -import { sfuPublisherQualityEvidenceFromMetrics } from "../src/client/media/sender-quality-evidence.ts"; import { SfuPublisher } from "../src/client/sfu/publisher.ts"; import { SfuSubscriber } from "../src/client/sfu/subscriber.ts"; -import type { ConnectionMetrics } from "../src/client/types.ts"; -import { mergeStatsReports } from "../src/client/webrtc/stats.ts"; -import { setCopy } from "../src/client/ui/copy.ts"; +import { + clientMessageSchema, + type SfuSignalMessage, +} from "../src/shared/protocol.ts"; -type EventHandler = (...args: unknown[]) => void; - -const livekit = vi.hoisted(() => { - class FakeSender { - track: MediaStreamTrack; - failNextSetParameters = false; - deferNextSetParameters = false; - private releaseSetParameters: (() => void) | null = null; - parameters: RTCRtpSendParameters = { - codecs: [], - encodings: [{}], - headerExtensions: [], - rtcp: { cname: "fake", reducedSize: true }, - transactionId: "fake", - }; - - constructor(track: MediaStreamTrack) { - this.track = track; - } - - readonly getParameters = vi.fn(() => this.parameters); - readonly setParameters = vi.fn( - async (parameters: RTCRtpSendParameters): Promise => { - if (this.failNextSetParameters) { - this.failNextSetParameters = false; - throw new Error("audio parameters rejected"); - } - const failure = state.nextSenderParameterError; - if (failure) { - state.nextSenderParameterError = null; - throw failure; - } - if (this.deferNextSetParameters) { - this.deferNextSetParameters = false; - await new Promise((resolve) => { - this.releaseSetParameters = resolve; - }); - } - this.parameters = parameters; - }, - ); - readonly getStats = vi.fn( - async (): Promise => - new Map() as unknown as RTCStatsReport, - ); - - releaseDeferredSetParameters(): void { - this.releaseSetParameters?.(); - this.releaseSetParameters = null; - } +class FakeTrack extends EventTarget { + readonly id = crypto.randomUUID(); + enabled = true; + contentHint = ""; + readyState = "live"; + readonly clones: FakeTrack[] = []; + readonly stop = vi.fn(() => { + this.readyState = "ended"; + }); + readonly getSettings = vi.fn(() => ({ + width: 1920, + height: 1080, + frameRate: 30, + })); + readonly applyConstraints = vi.fn( + async (_constraints: MediaTrackConstraints) => undefined, + ); + readonly clone = vi.fn(() => { + const next = new FakeTrack(this.kind); + this.clones.push(next); + return next; + }); + constructor(readonly kind: "audio" | "video") { + super(); } +} - class FakeLocalTrack { - currentTrack: MediaStreamTrack; - sender: FakeSender; - publishOptions?: Record; - savedDegradationPreference: RTCDegradationPreference | null = null; - - constructor(track: MediaStreamTrack) { - this.currentTrack = track; - this.sender = new FakeSender(track); - } - - get mediaStreamTrack(): MediaStreamTrack { - return this.currentTrack; - } - - readonly replaceTrack = vi.fn( - async (nextTrack: MediaStreamTrack): Promise => { - this.currentTrack = nextTrack; - this.sender.track = nextTrack; - }, - ); - - readonly setDegradationPreference = vi.fn( - async (preference: RTCDegradationPreference): Promise => { - this.savedDegradationPreference = preference; - }, - ); - - replaceSenderForTest(): FakeSender { - this.sender = new FakeSender(this.currentTrack); - return this.sender; - } - } - - class FakeLocalParticipant { - readonly publications: Array<{ - track: FakeLocalTrack; - videoTrack?: FakeLocalTrack; - audioTrack?: FakeLocalTrack; - rawTrack: MediaStreamTrack; - options: Record; - }> = []; - - readonly publishTrack = vi.fn( - async ( - rawTrack: MediaStreamTrack, - options: Record, - ) => { - const localTrack = new FakeLocalTrack(rawTrack); - localTrack.publishOptions = options; - const audioPreset = options.audioPreset; - if ( - rawTrack.kind === "audio" && - typeof audioPreset === "object" && - audioPreset !== null && - "maxBitrate" in audioPreset && - typeof audioPreset.maxBitrate === "number" - ) { - localTrack.sender.parameters.encodings = [ - { maxBitrate: audioPreset.maxBitrate }, - ]; - } - const screenShareEncoding = options.screenShareEncoding; - if ( - rawTrack.kind === "video" && - options.simulcast === false && - typeof screenShareEncoding === "object" && - screenShareEncoding !== null - ) { - const encoding = screenShareEncoding as { - maxBitrate?: number; - maxFramerate?: number; - }; - localTrack.sender.parameters.encodings = [ - { - maxBitrate: encoding.maxBitrate, - maxFramerate: encoding.maxFramerate, - scaleResolutionDownBy: 1, - }, - ]; - } - const publication = { - track: localTrack, - videoTrack: rawTrack.kind === "video" ? localTrack : undefined, - audioTrack: rawTrack.kind === "audio" ? localTrack : undefined, - rawTrack, - options, - }; - this.publications.push(publication); - return publication; - }, - ); - - getTrackPublication(source: string) { - return this.publications.find( - (publication) => publication.options.source === source, - ); - } - - async republishForReconnect(): Promise { - const previous = [...this.publications]; - this.publications.length = 0; - for (const publication of previous) { - delete publication.videoTrack; - delete publication.audioTrack; - await this.publishTrack(publication.rawTrack, { - ...publication.options, - }); - } - } - - republishedOptions: Array> = []; - readonly republishAllTracks = vi.fn(async (): Promise => { - this.republishedOptions = this.publications.map((publication) => ({ - ...publication.options, - })); - }); - - readonly unpublishTrack = vi.fn( - async (_track: FakeLocalTrack, _stopOnUnpublish: boolean) => undefined, - ); - } - - class FakeRemotePublication { - subscribed = false; - readonly setSubscribed = vi.fn((subscribed: boolean) => { - this.subscribed = subscribed; - }); - - constructor( - readonly trackSid: string, - readonly source: string, - ) {} - } - - class FakeRemoteParticipant { - readonly trackPublications = new Map(); - - constructor(readonly identity: string) {} - - add(publication: FakeRemotePublication): this { - this.trackPublications.set(publication.trackSid, publication); - return this; - } - } - - class FakeRoom { - readonly localParticipant = new FakeLocalParticipant(); - readonly remoteParticipants = new Map(); - readonly handlers = new Map(); - readonly connect = vi.fn( - async ( - _url: string, - _token: string, - _options: Record, - ): Promise => { - await state.connectGate; - }, - ); - readonly disconnect = vi.fn(async (_stopTracks?: boolean) => undefined); - - constructor(readonly options: Record = {}) { - state.rooms.push(this); - } - - on(event: string, handler: EventHandler): this { - const handlers = this.handlers.get(event) ?? []; - handlers.push(handler); - this.handlers.set(event, handlers); - return this; - } - - emit(event: string, ...args: unknown[]): void { - for (const handler of this.handlers.get(event) ?? []) { - handler(...args); - } - } - } - - const state: { - rooms: FakeRoom[]; - connectGate: Promise | null; - nextSenderParameterError: Error | null; - } = { - rooms: [], - connectGate: null, - nextSenderParameterError: null, - }; - - const AudioPresets = { - musicStereo: { maxBitrate: 64_000 }, - musicHighQualityStereo: { maxBitrate: 128_000 }, - } as const; - - return { - AudioPresets, - FakeRemoteParticipant, - FakeRemotePublication, - FakeRoom, - state, - }; -}); - -const RoomEvent = { - Disconnected: "disconnected", - ParticipantConnected: "participant-connected", - ParticipantDisconnected: "participant-disconnected", - Reconnected: "reconnected", - Reconnecting: "reconnecting", - TrackPublished: "track-published", - TrackSubscribed: "track-subscribed", - TrackUnpublished: "track-unpublished", - TrackUnsubscribed: "track-unsubscribed", -} as const; - -const Track = { - Source: { - Camera: "camera", - ScreenShare: "screen-share", - ScreenShareAudio: "screen-share-audio", - }, -} as const; - -vi.mock("livekit-client", () => ({ - AudioPresets: livekit.AudioPresets, - Room: livekit.FakeRoom, - RoomEvent, - Track, -})); - -class FakeMediaStream { - constructor(private readonly tracks: MediaStreamTrack[] = []) {} - - addTrack(track: MediaStreamTrack): void { - if (!this.tracks.includes(track)) this.tracks.push(track); - } - - removeTrack(track: MediaStreamTrack): void { - const index = this.tracks.indexOf(track); - if (index >= 0) this.tracks.splice(index, 1); - } - - getTracks(): MediaStreamTrack[] { +class FakeStream { + constructor(private tracks: FakeTrack[] = []) {} + getTracks(): FakeTrack[] { return [...this.tracks]; } - - getVideoTracks(): MediaStreamTrack[] { - return this.tracks.filter((item) => item.kind === "video"); + getVideoTracks(): FakeTrack[] { + return this.tracks.filter((track) => track.kind === "video"); } - - getAudioTracks(): MediaStreamTrack[] { - return this.tracks.filter((item) => item.kind === "audio"); + getAudioTracks(): FakeTrack[] { + return this.tracks.filter((track) => track.kind === "audio"); + } + addTrack(track: FakeTrack): void { + this.tracks.push(track); + } + removeTrack(track: FakeTrack): void { + this.tracks = this.tracks.filter((current) => current !== track); } } -const connection = { - url: "wss://sfu.example.test", - token: "short-lived-token", +class FakeSender { + readonly getParameters = vi.fn(() => structuredClone(this.parameters)); + readonly setParameters = vi.fn(async (parameters: RTCRtpSendParameters) => { + this.parameters = structuredClone(parameters); + }); + readonly replaceTrack = vi.fn(async (track: FakeTrack | null) => { + this.track = track; + }); + readonly getStats = vi.fn(async () => new Map()); + parameters: RTCRtpSendParameters; + constructor( + public track: FakeTrack | null, + encodings: RTCRtpEncodingParameters[], + ) { + this.parameters = { + encodings, + codecs: [], + headerExtensions: [], + rtcp: {}, + transactionId: "test", + }; + } + static getCapabilities(kind: string) { + return { + codecs: + kind === "audio" + ? [{ mimeType: "audio/opus", clockRate: 48000, channels: 2 }] + : [ + { mimeType: "video/VP8", clockRate: 90000 }, + { + mimeType: "video/H264", + clockRate: 90000, + sdpFmtpLine: "packetization-mode=1;profile-level-id=42e01f", + }, + { mimeType: "video/VP9", clockRate: 90000 }, + ], + }; + } +} + +class FakePc { + static instances: FakePc[] = []; + onicecandidate: ((event: { candidate: unknown }) => void) | null = null; + onconnectionstatechange: (() => void) | null = null; + ontrack: ((event: unknown) => void) | null = null; + connectionState: RTCPeerConnectionState = "new"; + signalingState: RTCSignalingState = "stable"; + localDescription: RTCSessionDescriptionInit | null = null; + remoteDescription: RTCSessionDescriptionInit | null = null; + readonly transceivers: Array<{ + sender: FakeSender; + setCodecPreferences: ReturnType; + }> = []; + report = new Map(); + readonly getStats = vi.fn(async () => this.report); + readonly close = vi.fn(() => { + this.connectionState = "closed"; + }); + readonly createOffer = vi.fn(async (_options?: RTCOfferOptions) => ({ + type: "offer" as const, + sdp: "offer", + })); + readonly createAnswer = vi.fn(async () => ({ + type: "answer" as const, + sdp: "answer", + })); + readonly setLocalDescription = vi.fn( + async (description: RTCSessionDescriptionInit) => { + this.localDescription = description; + this.signalingState = + description.type === "offer" ? "have-local-offer" : "stable"; + this.onicecandidate?.({ + candidate: { + protocol: "udp", + candidate: "candidate:1 1 UDP 1 127.0.0.1 4000 typ host", + toJSON: () => ({ + candidate: "candidate:1 1 UDP 1 127.0.0.1 4000 typ host", + }), + }, + }); + }, + ); + readonly setRemoteDescription = vi.fn( + async (description: RTCSessionDescriptionInit) => { + this.remoteDescription = description; + this.signalingState = + description.type === "offer" ? "have-remote-offer" : "stable"; + }, + ); + readonly addIceCandidate = vi.fn( + async (_candidate: RTCIceCandidateInit) => undefined, + ); + constructor(readonly configuration: RTCConfiguration) { + FakePc.instances.push(this); + } + addTransceiver(track: FakeTrack | string, options: RTCRtpTransceiverInit) { + const transceiver = { + sender: new FakeSender( + typeof track === "string" ? null : track, + options.sendEncodings ?? [{}], + ), + setCodecPreferences: vi.fn(), + }; + this.transceivers.push(transceiver); + return transceiver; + } + state(state: RTCPeerConnectionState): void { + this.connectionState = state; + this.onconnectionstatechange?.(); + } +} + +const config = { + revision: 4, + publicationGeneration: "publication_12345678", + connectionId: "connection_12345678", }; - -const qualityProfile = { - resolution: "1080p", - maxFramerate: 60, - maxBitrate: 8_000_000, - degradationPreference: "maintain-resolution", -} as const; - -function track(kind: "video" | "audio", id: string): MediaStreamTrack { - const mediaTrack = Object.assign(new EventTarget(), { - id, - kind, - contentHint: "", - enabled: true, - getSettings: () => ({}), - applyConstraints: vi.fn(async () => undefined), - stop: vi.fn(), - }) as unknown as MediaStreamTrack; - mediaTrack.clone = vi.fn(() => { - const clone = track(kind, id); - clone.getSettings = mediaTrack.getSettings.bind(mediaTrack); - return clone; - }); - return mediaTrack; -} - -function remoteTrack( - mediaStreamTrack: MediaStreamTrack, - getStats: () => RTCStatsReport | Promise = () => - new Map() as unknown as RTCStatsReport, -) { - return { - mediaStreamTrack, - getRTCStatsReport: vi.fn(async () => getStats()), - }; -} - -function statsReport( - records: Array & { id: string }>, -): RTCStatsReport { - return new Map( - records.map((record) => [record.id, record]), - ) as unknown as RTCStatsReport; -} - -function receiverReport(kind: "video" | "audio"): RTCStatsReport { - return statsReport([ - { - id: `${kind}-in`, - type: "inbound-rtp", - timestamp: 1_000, - kind, - trackIdentifier: `${kind}-1`, - estimatedPlayoutTimestamp: kind === "video" ? 10_000 : 10_012, - }, - ]); -} - -function senderReport( - trackId: string, - timestamp: number, - bytesSent: number, - framesEncoded: number, -): RTCStatsReport { - return statsReport([ - { - id: "video-out", - type: "outbound-rtp", - timestamp, - kind: "video", - transportId: "transport", - codecId: "codec", - mediaSourceId: "video-source", - bytesSent, - framesEncoded, - framesPerSecond: 57, - frameWidth: 1920, - frameHeight: 1080, - totalEncodeTime: framesEncoded * 0.004, - qualityLimitationReason: "bandwidth", - encoderImplementation: "ExternalEncoder", - powerEfficientEncoder: true, - }, - { - id: "video-source", - type: "media-source", - timestamp, - trackIdentifier: trackId, - framesPerSecond: 59, - }, - { - id: "transport", - type: "transport", - timestamp, - }, - { - id: "codec", - type: "codec", - timestamp, - transportId: "transport", - mimeType: "video/H264", - sdpFmtpLine: "profile-level-id=42e01f;packetization-mode=1", - }, - ]); -} - -function simulcastSenderReport( - trackId: string, - timestamp: number, - lowBytesSent: number, - highBytesSent: number, - highActive = true, -): RTCStatsReport { - const layer = ( - id: string, - rid: string, - width: number, - height: number, - bytesSent: number, - framesEncoded: number, - active = true, - ) => ({ - id, - type: "outbound-rtp", - timestamp, - kind: "video", - rid, - active, - mediaSourceId: "video-source", - bytesSent, - framesEncoded, - framesPerSecond: active ? 30 : 0, - frameWidth: width, - frameHeight: height, - qualityLimitationReason: "none", - qualityLimitationDurations: { - none: timestamp / 1_000, - bandwidth: 0, - cpu: 0, - other: 0, - }, - }); - return statsReport([ - layer("video-low", "q", 960, 540, lowBytesSent, timestamp / 40), - layer( - "video-high", - "f", - 1920, - 1080, - highBytesSent, - timestamp / 20, - highActive, - ), - { - id: "video-source", - type: "media-source", - timestamp, - trackIdentifier: trackId, - }, - ]); -} - -function audioSenderReport( - trackId: string, - timestamp: number, - bytesSent: number, -): RTCStatsReport { - return statsReport([ - { - id: "audio-out", - type: "outbound-rtp", - timestamp, - kind: "audio", - ssrc: 202, - transportId: "audio-transport", - mediaSourceId: "audio-source", - codecId: "audio-codec", - bytesSent, - }, - { - id: "audio-source", - type: "media-source", - timestamp, - kind: "audio", - trackIdentifier: trackId, - }, - { - id: "audio-transport", - type: "transport", - timestamp, - }, - { - id: "audio-codec", - type: "codec", - timestamp, - transportId: "audio-transport", - mimeType: "audio/opus", - }, - ]); -} - -function stream(...tracks: MediaStreamTrack[]): MediaStream { - return new FakeMediaStream(tracks) as unknown as MediaStream; -} - -function deferred(): { - promise: Promise; - resolve: () => void; -} { - let resolve = (): void => undefined; - const promise = new Promise((next) => { - resolve = next; - }); - return { promise, resolve }; -} +const signal = (payload: Partial): SfuSignalMessage => ({ + type: "sfu-signal", + ...config, + kind: "description", + ...payload, +}); +const cleanups: Array<() => Promise> = []; +const stream = (...tracks: FakeTrack[]): MediaStream => + new FakeStream(tracks) as unknown as MediaStream; beforeEach(() => { - setCopy({ lang: "zh", vis: false }); - livekit.state.rooms.length = 0; - livekit.state.connectGate = null; - livekit.state.nextSenderParameterError = null; - vi.stubGlobal("MediaStream", FakeMediaStream); + vi.useFakeTimers(); + FakePc.instances = []; + vi.stubGlobal("MediaStream", FakeStream); + vi.stubGlobal("RTCPeerConnection", FakePc); + vi.stubGlobal("RTCRtpSender", FakeSender); }); - -afterEach(() => { - vi.clearAllTimers(); +afterEach(async () => { + for (const cleanup of cleanups.splice(0)) await cleanup(); vi.useRealTimers(); vi.unstubAllGlobals(); }); -describe("SfuPublisher", () => { - it("reports the aggregate bitrate of active simulcast layers", async () => { - vi.useFakeTimers(); - const updates: Array = []; - const publisher = new SfuPublisher({ - onStats: (metrics) => updates.push(metrics), - }); - const video = track("video", "video-1"); - await publisher.connect(connection); - await publisher.activate(stream(video), qualityProfile); - const sender = livekit.state.rooms[0].localParticipant.publications[0].track - .sender; - sender.parameters.encodings = [ - { rid: "q", active: true }, - { rid: "f", active: true }, - ]; - sender.getStats - .mockResolvedValueOnce( - simulcastSenderReport(video.id, 1_000, 100_000, 200_000), - ) - .mockResolvedValueOnce( - simulcastSenderReport(video.id, 3_000, 300_000, 600_000), - ); - - await vi.advanceTimersByTimeAsync(2_000); - expect(updates.at(-1)?.bitrateKbps).toBeNull(); - expect(updates.at(-1)).toMatchObject({ - rtpRid: "f", - frameWidth: 1920, - frameHeight: 1080, - videoEncodingCount: 2, - activeVideoEncodingCount: 2, - }); - expect( - sfuPublisherQualityEvidenceFromMetrics( - updates.at(-1)!, - 7, - "simulcast_publication_12345678", - ), - ).toMatchObject({ state: "unknown", sampleTimestampMs: null }); - await vi.advanceTimersByTimeAsync(2_000); - expect(updates.at(-1)?.bitrateKbps).toBe(2_400); - expect(updates.at(-1)?.sampleTimestampMs).toBe(3_000); - expect( - sfuPublisherQualityEvidenceFromMetrics( - updates.at(-1)!, - 7, - "simulcast_publication_12345678", - ), - ).toMatchObject({ - state: "healthy", - sampleTimestampMs: 3_000, - diagnostics: { - videoEncodingCount: 2, - activeVideoEncodingCount: 2, - }, - }); - }); - - it("reports the highest active SFU representation", async () => { - vi.useFakeTimers(); - const updates: Array = []; - const publisher = new SfuPublisher({ - onStats: (metrics) => updates.push(metrics), - }); - const video = track("video", "video-1"); - await publisher.connect(connection); - await publisher.activate(stream(video), qualityProfile); - const sender = livekit.state.rooms[0].localParticipant.publications[0].track - .sender; - sender.parameters.encodings = [ - { rid: "q", active: true }, - { rid: "f", active: false }, - ]; - sender.getStats.mockResolvedValueOnce( - simulcastSenderReport(video.id, 1_000, 100_000, 200_000, false), - ); - - await vi.advanceTimersByTimeAsync(2_000); - - expect(updates.at(-1)).toMatchObject({ - rtpRid: "q", - frameWidth: 960, - frameHeight: 540, - videoEncodingCount: 2, - activeVideoEncodingCount: 1, - }); - }); - - it("enables balanced SFU adaptation after startup frames", async () => { - vi.useFakeTimers(); - const publisher = new SfuPublisher(); - const video = track("video", "startup-video"); - const balancedProfile = { - ...qualityProfile, - degradationPreference: "balanced", - } as const; - - await publisher.connect(connection); - await publisher.activate(stream(video), balancedProfile); - const localTrack = livekit.state.rooms[0].localParticipant.publications[0] - .track; - const sender = localTrack.sender; - expect(localTrack.savedDegradationPreference).toBe("maintain-resolution"); - expect(sender.parameters.degradationPreference).toBe( - "maintain-resolution", - ); - - sender.getStats - .mockResolvedValueOnce(senderReport(video.id, 1_000, 10_000, 4)) - .mockResolvedValueOnce(senderReport(video.id, 2_000, 20_000, 5)); - await vi.advanceTimersByTimeAsync(2_000); - expect(localTrack.savedDegradationPreference).toBe("maintain-resolution"); - expect(sender.setParameters).toHaveBeenCalledOnce(); - - await vi.advanceTimersByTimeAsync(2_000); - await vi.waitFor(() => - expect(localTrack.savedDegradationPreference).toBe("balanced"), - ); - expect(sender.parameters.degradationPreference).toBe("balanced"); - expect(sender.setParameters).toHaveBeenCalledTimes(2); - }); - - it("samples only the current published sender and clears replaced evidence", async () => { - vi.useFakeTimers(); - const updates: Array = []; - const publisher = new SfuPublisher({ - onStats: (metrics) => updates.push(metrics), - }); - const previousVideo = track("video", "video-1"); - previousVideo.getSettings = () => ({ - width: 1920, - height: 1080, - frameRate: 60, - }); - await publisher.connect(connection); - await publisher.activate(stream(previousVideo), qualityProfile); - const sender = livekit.state.rooms[0].localParticipant.publications[0].track - .sender; - sender.getStats.mockResolvedValueOnce( - senderReport(previousVideo.id, 1_000, 100_000, 60), - ); - - await vi.advanceTimersByTimeAsync(2_000); - expect(updates.at(-1)).toMatchObject({ - captureWidth: 1920, - captureHeight: 1080, - captureFramesPerSecond: 60, - trackIdentifier: previousVideo.id, - rtpRid: null, - mediaSourceFramesPerSecond: 59, - framesPerSecond: 57, - resolution: "1920x1080", - codec: "video/H264", - codecProfile: "profile-level-id=42e01f", - encoderImplementation: "ExternalEncoder", - powerEfficientEncoder: true, - qualityLimitationReason: "bandwidth", - nativeEdgeQualityState: "unknown", - }); - - sender.getStats.mockResolvedValueOnce( - senderReport(previousVideo.id, 3_000, 300_000, 180), - ); - await vi.advanceTimersByTimeAsync(2_000); - expect(updates.at(-1)).toMatchObject({ - bitrateKbps: 800, - intervalFramesEncoded: 120, - intervalEncodeTimeMs: 480, - intervalEncodeMs: 4, - }); - - let resolveOld!: (report: RTCStatsReport) => void; - sender.getStats.mockImplementationOnce( - () => new Promise((resolve) => { resolveOld = resolve; }), - ); - await vi.advanceTimersByTimeAsync(2_000); - const nextVideo = track("video", "video-2"); - nextVideo.getSettings = () => ({ - width: 1280, - height: 720, - frameRate: 30, - }); - await expect(publisher.replaceStream(stream(nextVideo))).resolves.toBe(true); - expect(updates.at(-1)).toBeNull(); - resolveOld(senderReport(previousVideo.id, 5_000, 500_000, 300)); - await Promise.resolve(); - expect(updates.at(-1)).toBeNull(); - - sender.getStats.mockResolvedValueOnce( - senderReport(nextVideo.id, 4_000, 400_000, 240), - ); - await vi.advanceTimersByTimeAsync(2_000); - expect(updates.at(-1)?.trackIdentifier).toBe(nextVideo.id); - - const updateCount = updates.length; - const localTrack = livekit.state.rooms[0].localParticipant.publications[0] - .track; - const replacementSender = localTrack.replaceSenderForTest(); - replacementSender.getStats.mockResolvedValueOnce( - senderReport(nextVideo.id, 6_000, 500_000, 270), - ); - await vi.advanceTimersByTimeAsync(2_000); - expect(updates[updateCount]).toBeNull(); - expect(updates.at(-1)).toMatchObject({ - trackIdentifier: nextVideo.id, - framesPerSecond: 57, - }); - - await publisher.deactivate(); - expect(updates.at(-1)).toBeNull(); - }); - - it("merges the current audio sender report and resets after its sender changes", async () => { - vi.useFakeTimers(); - const updates: Array = []; - const publisher = new SfuPublisher({ - onStats: (metrics) => updates.push(metrics), - }); - const video = track("video", "video-1"); - const audio = track("audio", "audio-1"); - await publisher.connect(connection); - await publisher.activate(stream(video, audio), qualityProfile); - const room = livekit.state.rooms[0]; - const videoSender = room.localParticipant.publications[0].track.sender; - const audioTrack = room.localParticipant.publications[1].track; - const audioSender = audioTrack.sender; - videoSender.getStats - .mockResolvedValueOnce(senderReport(video.id, 1_000, 100_000, 60)) - .mockResolvedValueOnce(senderReport(video.id, 3_000, 300_000, 180)); - audioSender.getStats - .mockResolvedValueOnce(audioSenderReport(audio.id, 1_000, 10_000)) - .mockResolvedValueOnce(audioSenderReport(audio.id, 3_000, 50_000)); - - await vi.advanceTimersByTimeAsync(2_000); - expect(updates.at(-1)).toMatchObject({ - audioBitrateKbps: null, - audioCodec: "audio/opus", - }); - await vi.advanceTimersByTimeAsync(2_000); - expect(updates.at(-1)).toMatchObject({ - audioBitrateKbps: 160, - audioCodec: "audio/opus", - }); - - const replacementSender = audioTrack.replaceSenderForTest(); - videoSender.getStats - .mockResolvedValueOnce(senderReport(video.id, 5_000, 500_000, 300)) - .mockResolvedValueOnce(senderReport(video.id, 7_000, 700_000, 420)); - replacementSender.getStats - .mockResolvedValueOnce(audioSenderReport(audio.id, 5_000, 90_000)) - .mockResolvedValueOnce(audioSenderReport(audio.id, 7_000, 130_000)); - - await vi.advanceTimersByTimeAsync(2_000); - expect(updates.at(-2)).toBeNull(); - expect(updates.at(-1)?.audioBitrateKbps).toBeNull(); - await vi.advanceTimersByTimeAsync(2_000); - expect(updates.at(-1)?.audioBitrateKbps).toBe(160); - expect(audioSender.getStats).toHaveBeenCalledTimes(2); - expect(replacementSender.getStats).toHaveBeenCalledTimes(2); - }); - - it("enables Dynacast while preparing SFU fallback", async () => { - const publisher = new SfuPublisher(); - - await expect(publisher.connect(connection)).resolves.toBe(true); - - const room = livekit.state.rooms[0]; - expect(room.options).toEqual({ - disconnectOnPageLeave: false, - dynacast: true, - stopLocalTrackOnUnpublish: false, - }); - expect(room.connect).toHaveBeenCalledWith(connection.url, connection.token, { - autoSubscribe: false, - rtcConfig: { iceServers: [] }, - }); - expect(room.localParticipant.publishTrack).not.toHaveBeenCalled(); - - await publisher.disconnect(); - await publisher.disconnect(); - expect(room.disconnect).toHaveBeenCalledOnce(); - expect(room.disconnect).toHaveBeenCalledWith(false); - }); - - it("retains the bounded stage after connection setup fails", async () => { - livekit.state.connectGate = Promise.reject(new Error("connect failed")); - const publisher = new SfuPublisher(); - - await expect(publisher.connect(connection)).rejects.toThrow("connect failed"); - - expect(publisher.getFailureStage()).toBe("connect"); - expect(livekit.state.rooms[0]?.disconnect).toHaveBeenCalledWith(false); - }); - - it("leaves VP8 screen-share simulcast layers to LiveKit defaults", async () => { - const publisher = new SfuPublisher(); - const video = track("video", "video-1"); - const audio = track("audio", "audio-1"); - await publisher.connect(connection); - - await expect( - publisher.activate(stream(video, audio), qualityProfile), - ).resolves.toBe(true); - - const room = livekit.state.rooms[0]; - const sender = room.localParticipant.publications[0].track.sender; - const publishedVideo = room.localParticipant.publishTrack.mock.calls[0]?.[0]; - expect(publishedVideo).not.toBe(video); - expect(publishedVideo?.id).toBe(video.id); - expect(room.localParticipant.publishTrack).toHaveBeenNthCalledWith(1, publishedVideo, { - source: Track.Source.ScreenShare, - backupCodec: false, - videoCodec: "vp8", - screenShareEncoding: { - maxBitrate: 8_000_000, - maxFramerate: 60, - }, - degradationPreference: "maintain-resolution", - }); - expect( - room.localParticipant.publishTrack.mock.calls[0]?.[1], - ).not.toHaveProperty("simulcast"); - expect(room.localParticipant.publishTrack).toHaveBeenNthCalledWith(2, audio, { - source: Track.Source.ScreenShareAudio, - audioPreset: { maxBitrate: 128_000 }, - forceStereo: true, - dtx: false, - red: false, - }); - expect(sender.setParameters).toHaveBeenCalledOnce(); - expect(sender.parameters.encodings).toEqual([ - { - maxBitrate: 8_000_000, - maxFramerate: 60, - scaleResolutionDownBy: 1, - }, - ]); - expect(sender.parameters.encodings[0]).not.toHaveProperty("rid"); - expect(publisher.getSenderParameters()).toEqual({ - requested: { - maxBitrate: 8_000_000, - maxFramerate: 60, - scaleResolutionDownBy: 1, - degradationPreference: "maintain-resolution", - scalabilityMode: null, - }, - applied: { - maxBitrate: 8_000_000, - maxFramerate: 60, - scaleResolutionDownBy: 1, - degradationPreference: "maintain-resolution", - scalabilityMode: null, - }, - mismatches: [], - }); - - await expect(publisher.deactivate()).resolves.toBe(true); - expect(room.localParticipant.unpublishTrack).toHaveBeenCalledTimes(2); - expect(publishedVideo?.stop).toHaveBeenCalledOnce(); - expect(video.stop).not.toHaveBeenCalled(); - expect(room.disconnect).not.toHaveBeenCalled(); - }); - - it("owns SFU video clone controls without stopping the capture source", async () => { - const publisher = new SfuPublisher(); - const sourceVideo = track("video", "capture-video"); - sourceVideo.contentHint = "motion"; - await publisher.connect(connection); - await publisher.activate(stream(sourceVideo), qualityProfile); - const publishedVideo = livekit.state.rooms[0].localParticipant.publications[0] - .rawTrack; - - expect(publishedVideo).not.toBe(sourceVideo); - expect(publishedVideo.contentHint).toBe("motion"); - publisher.setPaused(true); - expect(publishedVideo.enabled).toBe(false); - expect(sourceVideo.enabled).toBe(true); - publisher.setPaused(false); - - await expect( - publisher.updateProfile(QUALITY_PROFILES["720p30"]), - ).resolves.toBe(true); - expect(publishedVideo.applyConstraints).toHaveBeenLastCalledWith({ - width: { ideal: 1280, max: 1280 }, - height: { ideal: 720, max: 720 }, - frameRate: { ideal: 30, max: 30 }, - }); - - await publisher.disconnect(); - expect(publishedVideo.stop).toHaveBeenCalledOnce(); - expect(sourceVideo.stop).not.toHaveBeenCalled(); - }); - - it.each([ - ["saver", 64_000], - ["music", 128_000], - ["very-high", 192_000], - ] as const)("maps the %s audio preset to %i bps", async (screenAudioQuality, bitrate) => { - const publisher = new SfuPublisher(); - const video = track("video", `video-${screenAudioQuality}`); - const audio = track("audio", `audio-${screenAudioQuality}`); - await publisher.connect(connection); - - await expect( - publisher.activate(stream(video, audio), { - ...qualityProfile, - screenAudioQuality, - }), - ).resolves.toBe(true); - - expect( - livekit.state.rooms[0]?.localParticipant.publishTrack, - ).toHaveBeenNthCalledWith(2, audio, { - source: Track.Source.ScreenShareAudio, - audioPreset: { maxBitrate: bitrate }, - forceStereo: true, - dtx: false, - red: false, - }); - }); - - it("fails closed when initial sender configuration is rejected", async () => { - const disconnected = vi.fn(); - const publisher = new SfuPublisher({ onDisconnected: disconnected }); - const sourceVideo = track("video", "video-1"); - await publisher.connect(connection); - livekit.state.nextSenderParameterError = new Error("parameters rejected"); - - await expect( - publisher.activate(stream(sourceVideo), qualityProfile), - ).rejects.toThrow("parameters rejected"); - - const publishedVideo = livekit.state.rooms[0].localParticipant.publications[0] - .rawTrack; - expect(publishedVideo.stop).toHaveBeenCalledOnce(); - expect(sourceVideo.stop).not.toHaveBeenCalled(); - expect(livekit.state.rooms[0].disconnect).toHaveBeenCalledWith(false); - expect(disconnected).toHaveBeenCalledOnce(); - expect(publisher.getSenderParameters()).toBeNull(); - expect(publisher.getFailureStage()).toBe("sender-config"); - }); - - it("records an unexpected active transport disconnect", async () => { - const disconnected = vi.fn(); - const publisher = new SfuPublisher({ onDisconnected: disconnected }); - await publisher.connect(connection); - await publisher.activate(stream(track("video", "video-1")), qualityProfile); - - livekit.state.rooms[0].emit(RoomEvent.Disconnected); - - expect(publisher.getFailureStage()).toBe("transport"); - expect(disconnected).toHaveBeenCalledOnce(); - }); - - it("updates the active sender profile without republishing", async () => { - const publisher = new SfuPublisher(); - await publisher.connect(connection); - await publisher.activate(stream(track("video", "video-1")), qualityProfile); - const room = livekit.state.rooms[0]; - const videoPublication = room.localParticipant.publications[0]; - - await expect( - publisher.updateProfile({ - resolution: "720p", - maxFramerate: 30, - maxBitrate: 3_000_000, - degradationPreference: "balanced", - }), - ).resolves.toBe(true); - - expect(videoPublication.track.sender.setParameters).toHaveBeenCalledWith( - expect.objectContaining({ - degradationPreference: "balanced", - encodings: [ - expect.objectContaining({ - maxBitrate: 3_000_000, - maxFramerate: 30, - scaleResolutionDownBy: 1, - }), - ], - }), - ); - expect(room.localParticipant.publishTrack).toHaveBeenCalledOnce(); - expect(publisher.getQualityWarning()).toBeNull(); - }); - - it("updates an active audio sender and retained options without republishing", async () => { - const publisher = new SfuPublisher(); - await publisher.connect(connection); - await publisher.activate( - stream(track("video", "video-1"), track("audio", "audio-1")), - { ...qualityProfile, screenAudioQuality: "saver" }, - ); - const room = livekit.state.rooms[0]; - const videoPublication = room.localParticipant.publications[0]; - const audioPublication = room.localParticipant.publications[1]; - - await expect( - publisher.updateProfile({ - ...qualityProfile, - screenAudioQuality: "very-high", - }), - ).resolves.toBe(true); - - expect(videoPublication.track.sender.setParameters).toHaveBeenCalledOnce(); - expect(audioPublication.track.sender.setParameters).toHaveBeenCalledTimes(2); - expect(audioPublication.track.sender.parameters.encodings[0]?.maxBitrate).toBe( - 192_000, - ); - expect(audioPublication.options).toMatchObject({ - audioPreset: { maxBitrate: 192_000 }, - forceStereo: true, - dtx: false, - red: false, - }); - expect(publisher.getAudioSenderParameters()).toEqual({ - requestedMaxBitrate: 192_000, - appliedMaxBitrate: 192_000, - mismatch: false, - }); - expect(room.localParticipant.publishTrack).toHaveBeenCalledTimes(2); - - await room.localParticipant.republishAllTracks(); - expect(room.localParticipant.republishedOptions[1]).toMatchObject({ - audioPreset: { maxBitrate: 192_000 }, - forceStereo: true, - dtx: false, - red: false, - }); - }); - - it("publishes one H264 source without a backup codec", async () => { - const publisher = new SfuPublisher(); - const video = track("video", "video-h264"); - await publisher.connect(connection); - - await expect( - publisher.activate(stream(video), qualityProfile, "h264"), - ).resolves.toBe(true); - - const publication = livekit.state.rooms[0].localParticipant.publications[0]; - expect(publication.options).toMatchObject({ - source: Track.Source.ScreenShare, - backupCodec: false, - videoCodec: "h264", - }); - await expect( - publisher.updateProfile(QUALITY_PROFILES["720p30"]), - ).resolves.toBe(true); - expect(publication.options.videoCodec).toBe("h264"); - }); - - it("keeps media and prior audio readback when a live update fails", async () => { - const publisher = new SfuPublisher(); - await publisher.connect(connection); - await publisher.activate( - stream(track("video", "video-1"), track("audio", "audio-1")), - { ...qualityProfile, screenAudioQuality: "music" }, - ); - const room = livekit.state.rooms[0]; - const videoPublication = room.localParticipant.publications[0]; - const audioPublication = room.localParticipant.publications[1]; - audioPublication.track.sender.failNextSetParameters = true; - - await expect( - publisher.updateProfile({ - ...qualityProfile, - screenAudioQuality: "very-high", - }), - ).resolves.toBe(false); - - expect(publisher.getAudioSenderParameters()?.appliedMaxBitrate).toBe( - 128_000, - ); - expect(publisher.getQualityWarning()).toContain( - "应用 SFU 音频发送参数失败", - ); - expect(audioPublication.track.sender.parameters.encodings[0]?.maxBitrate).toBe( - 128_000, - ); - expect(audioPublication.options).toMatchObject({ - audioPreset: { maxBitrate: 192_000 }, - }); - expect(videoPublication.track.sender.setParameters).toHaveBeenCalledOnce(); - expect(room.disconnect).not.toHaveBeenCalled(); - expect(room.localParticipant.publishTrack).toHaveBeenCalledTimes(2); - - await expect( - publisher.updateProfile({ - ...qualityProfile, - screenAudioQuality: "very-high", - }), - ).resolves.toBe(true); - expect(publisher.getAudioSenderParameters()?.appliedMaxBitrate).toBe( - 192_000, - ); - expect(publisher.getQualityWarning()).toBeNull(); - }); - - it("keeps rapid SFU audio updates last-wins", async () => { - const publisher = new SfuPublisher(); - await publisher.connect(connection); - await publisher.activate( - stream(track("video", "video-1"), track("audio", "audio-1")), - { ...qualityProfile, screenAudioQuality: "music" }, - ); - const room = livekit.state.rooms[0]; - const audioPublication = room.localParticipant.publications[1]; - const audioSender = audioPublication.track.sender; - audioSender.deferNextSetParameters = true; - - const saver = publisher.updateProfile({ - ...qualityProfile, - screenAudioQuality: "saver", - }); - await vi.waitFor(() => - expect(audioSender.setParameters).toHaveBeenCalledTimes(2), - ); - const music = publisher.updateProfile({ - ...qualityProfile, - screenAudioQuality: "music", - }); - expect(audioPublication.options).toMatchObject({ - audioPreset: { maxBitrate: 128_000 }, - }); - audioSender.releaseDeferredSetParameters(); - - await expect(saver).resolves.toBe(false); - await expect(music).resolves.toBe(true); - expect(audioSender.parameters.encodings[0]?.maxBitrate).toBe(128_000); - expect(publisher.getAudioSenderParameters()?.appliedMaxBitrate).toBe( - 128_000, - ); - expect(room.localParticipant.republishAllTracks).not.toHaveBeenCalled(); - }); - - it("reapplies the latest audio ceiling to a replacement sender after reconnect", async () => { - const publisher = new SfuPublisher(); - await publisher.connect(connection); - await publisher.activate( - stream(track("video", "video-1"), track("audio", "audio-1")), - { ...qualityProfile, screenAudioQuality: "music" }, - ); - const room = livekit.state.rooms[0]; - const audioPublication = room.localParticipant.publications[1]; - const oldSender = audioPublication.track.sender; - oldSender.deferNextSetParameters = true; - - const updating = publisher.updateProfile({ - ...qualityProfile, - screenAudioQuality: "very-high", - }); - await vi.waitFor(() => - expect(oldSender.setParameters).toHaveBeenCalledTimes(2), - ); - const replacementSender = audioPublication.track.replaceSenderForTest(); - - room.emit(RoomEvent.Reconnected); - oldSender.releaseDeferredSetParameters(); - - await expect(updating).resolves.toBe(false); - await vi.waitFor(() => - expect(replacementSender.setParameters).toHaveBeenCalledOnce(), - ); - expect(replacementSender.parameters.encodings[0]?.maxBitrate).toBe(192_000); - expect(publisher.getAudioSenderParameters()?.appliedMaxBitrate).toBe( - 192_000, - ); - expect(room.localParticipant.republishAllTracks).not.toHaveBeenCalled(); - }); - - it("rebinds republished tracks after a full reconnect", async () => { - const publisher = new SfuPublisher(); - const sourceVideo = track("video", "video-1"); - await publisher.connect(connection); - await publisher.activate( - stream(sourceVideo, track("audio", "audio-1")), - { ...qualityProfile, screenAudioQuality: "music" }, - ); - await expect( - publisher.updateProfile({ - ...qualityProfile, - screenAudioQuality: "very-high", - }), - ).resolves.toBe(true); - const room = livekit.state.rooms[0]; - const oldVideoPublication = room.localParticipant.publications[0]; - const oldAudioPublication = room.localParticipant.publications[1]; - const oldPublishedVideo = oldVideoPublication.rawTrack; - - await room.localParticipant.republishForReconnect(); - const videoPublication = room.localParticipant.getTrackPublication( - Track.Source.ScreenShare, - )!; - const audioPublication = room.localParticipant.getTrackPublication( - Track.Source.ScreenShareAudio, - )!; - expect(videoPublication).not.toBe(oldVideoPublication); - expect(audioPublication).not.toBe(oldAudioPublication); - - room.emit(RoomEvent.Reconnected); - - await vi.waitFor(() => - expect(videoPublication.track.currentTrack).not.toBe(oldPublishedVideo), - ); - expect(publisher.getAudioSenderParameters()).not.toBeNull(); - expect(videoPublication.track.currentTrack.id).toBe(sourceVideo.id); - expect(oldPublishedVideo.stop).toHaveBeenCalledOnce(); - expect(sourceVideo.stop).not.toHaveBeenCalled(); - expect(audioPublication.track.sender.setParameters).not.toHaveBeenCalled(); - expect(audioPublication.track.sender.parameters.encodings[0]?.maxBitrate).toBe( - 192_000, - ); - expect(publisher.getAudioSenderParameters()?.appliedMaxBitrate).toBe( - 192_000, - ); - expect(publisher.getSenderParameters()).toBeNull(); - expect(publisher.getQualityWarning()).toBeNull(); - }); - - it("reconciles pause state onto a video clone committed after reconnect", async () => { - const publisher = new SfuPublisher(); - const sourceVideo = track("video", "video-pause-race"); - await publisher.connect(connection); - await publisher.activate(stream(sourceVideo), qualityProfile); - publisher.setPaused(true); - sourceVideo.enabled = false; - - const room = livekit.state.rooms[0]; - await room.localParticipant.republishForReconnect(); - const publication = room.localParticipant.getTrackPublication( - Track.Source.ScreenShare, - )!; - const replaceGate = deferred(); - publication.track.replaceTrack.mockImplementationOnce( - async (nextTrack: MediaStreamTrack) => { - await replaceGate.promise; - publication.track.currentTrack = nextTrack; - publication.track.sender.track = nextTrack; - }, - ); - - room.emit(RoomEvent.Reconnected); - await vi.waitFor(() => - expect(publication.track.replaceTrack).toHaveBeenCalledOnce(), - ); - sourceVideo.enabled = true; - publisher.setPaused(false); - replaceGate.resolve(); - - await vi.waitFor(() => - expect(publication.track.currentTrack).not.toBe(publication.rawTrack), - ); - expect(publication.track.currentTrack.enabled).toBe(true); - }); - - it("retains video sender readback across a signal-only reconnect", async () => { - const publisher = new SfuPublisher(); - await publisher.connect(connection); - await publisher.activate( - stream(track("video", "video-1"), track("audio", "audio-1")), - { ...qualityProfile, screenAudioQuality: "music" }, - ); - const room = livekit.state.rooms[0]; - const videoSender = room.localParticipant.publications[0].track.sender; - const audioSender = room.localParticipant.publications[1].track.sender; - videoSender.setParameters.mockImplementationOnce(async (parameters) => { - videoSender.parameters = { - ...parameters, - encodings: parameters.encodings.map((encoding) => ({ - ...encoding, - maxBitrate: 2_000_000, - })), - }; - }); - await expect( - publisher.updateProfile({ - ...qualityProfile, - maxBitrate: 5_000_000, - degradationPreference: "balanced", - screenAudioQuality: "music", - }), - ).resolves.toBe(true); - const senderParameters = publisher.getSenderParameters(); - const qualityWarning = publisher.getQualityWarning(); - expect(senderParameters).not.toBeNull(); - expect(qualityWarning).not.toBeNull(); - const audioUpdates = audioSender.setParameters.mock.calls.length; - - room.emit(RoomEvent.Reconnected); - - await vi.waitFor(() => - expect(publisher.getAudioSenderParameters()?.appliedMaxBitrate).toBe( - 128_000, - ), - ); - expect(audioSender.setParameters).toHaveBeenCalledTimes(audioUpdates); - expect(publisher.getSenderParameters()).toBe(senderParameters); - expect(publisher.getQualityWarning()).toBe(qualityWarning); - expect(room.disconnect).not.toHaveBeenCalled(); - }); - - it("keeps publisher stats continuous across a signal-only reconnect", async () => { - vi.useFakeTimers(); - const updates: Array = []; - const publisher = new SfuPublisher({ - onStats: (metrics) => updates.push(metrics), - }); - const video = track("video", "video-1"); - await publisher.connect(connection); - await publisher.activate(stream(video), qualityProfile); - const room = livekit.state.rooms[0]; - const sender = room.localParticipant.publications[0].track.sender; - sender.getStats - .mockResolvedValueOnce(senderReport(video.id, 1_000, 100_000, 60)) - .mockResolvedValueOnce(senderReport(video.id, 3_000, 300_000, 180)) - .mockResolvedValueOnce(senderReport(video.id, 5_000, 500_000, 300)); - - await vi.advanceTimersByTimeAsync(4_000); - const resets = updates.filter((update) => update === null).length; - room.emit(RoomEvent.Reconnected); - await Promise.resolve(); - await vi.advanceTimersByTimeAsync(2_000); - - expect(updates.filter((update) => update === null)).toHaveLength(resets); - expect(updates.at(-1)).toMatchObject({ - bitrateKbps: 800, - intervalFramesEncoded: 120, - }); - }); - - it.each(["publication", "sender"] as const)( - "fails the SFU route when republished audio loses its %s", - async (missing) => { - const disconnected = vi.fn(); - const publisher = new SfuPublisher({ onDisconnected: disconnected }); - await publisher.connect(connection); - await publisher.activate( - stream(track("video", "video-1"), track("audio", "audio-1")), - { ...qualityProfile, screenAudioQuality: "music" }, - ); - const room = livekit.state.rooms[0]; - await room.localParticipant.republishForReconnect(); - const audioPublication = room.localParticipant.getTrackPublication( - Track.Source.ScreenShareAudio, - )!; - if (missing === "publication") { - room.localParticipant.publications.splice( - room.localParticipant.publications.indexOf(audioPublication), - 1, - ); - } else { - delete audioPublication.audioTrack; - } - - room.emit(RoomEvent.Reconnected); - - await vi.waitFor(() => expect(room.disconnect).toHaveBeenCalledWith(false)); - expect(publisher.getFailureStage()).toBe("transport"); - expect(disconnected).toHaveBeenCalledOnce(); - }, +async function publisher(audio = true) { + const send = vi.fn((_message: SfuSignalMessage) => true); + const onDisconnected = vi.fn(); + const video = new FakeTrack("video"); + const sound = new FakeTrack("audio"); + const publisher = new SfuPublisher({ send, onDisconnected }); + cleanups.push(() => publisher.disconnect()); + await publisher.connect(config); + await publisher.activate( + stream(video, ...(audio ? [sound] : [])), + QUALITY_PROFILES["1080p30"], + "h264", ); + return { + publisher, + send, + onDisconnected, + video, + sound, + pc: FakePc.instances[0]!, + }; +} - it("retains the active profile for LiveKit track restart and republish", async () => { - const publisher = new SfuPublisher(); - await publisher.connect(connection); - await publisher.activate(stream(track("video", "video-1")), qualityProfile); - const publication = livekit.state.rooms[0].localParticipant.publications[0]; - const localTrack = publication.track; - - await expect( - publisher.updateProfile({ - resolution: "720p", - maxFramerate: 30, - maxBitrate: 3_000_000, - degradationPreference: "maintain-framerate", - }), - ).resolves.toBe(true); - - expect(localTrack.setDegradationPreference).toHaveBeenLastCalledWith( - "maintain-framerate", - ); - expect( - localTrack.setDegradationPreference.mock.invocationCallOrder.at(-1)!, - ).toBeLessThan( - localTrack.sender.setParameters.mock.invocationCallOrder.at(-1)!, - ); - expect(localTrack.savedDegradationPreference).toBe("maintain-framerate"); - expect(localTrack.sender.parameters).toMatchObject({ - degradationPreference: "maintain-framerate", - encodings: [ - { - maxBitrate: 3_000_000, - maxFramerate: 30, - scaleResolutionDownBy: 1, - }, +describe("embedded SFU browser transport", () => { + it("publishes one bounded simulcast PC, offers before ICE and admits only the selected codec", async () => { + const { publisher: host, pc, send, video } = await publisher(); + expect(pc.configuration).toEqual({ iceServers: [] }); + expect(pc.transceivers).toHaveLength(2); + expect(pc.transceivers[0]!.sender.track).toBe(video.clones[0]); + const offer = send.mock.calls[0]![0]; + expect(offer.kind).toBe("description"); + expect(offer.media).toEqual({ + codec: "h264", + layers: [ + { rid: "q", width: 960, height: 540, bitrate: 1_250_000 }, + { rid: "h", width: 1920, height: 1080, bitrate: 5_000_000 }, ], + audio: true, + audioBitrate: 128_000, }); - expect(localTrack.publishOptions).toBe(publication.options); - expect(publication.options).toMatchObject({ - source: Track.Source.ScreenShare, - backupCodec: false, - screenShareEncoding: { - maxBitrate: 3_000_000, - maxFramerate: 30, - }, - degradationPreference: "maintain-framerate", - }); - - await livekit.state.rooms[0].localParticipant.republishAllTracks(); - expect( - livekit.state.rooms[0].localParticipant.republishedOptions[0], - ).toMatchObject({ - screenShareEncoding: { - maxBitrate: 3_000_000, - maxFramerate: 30, - }, - degradationPreference: "maintain-framerate", - }); - }); - - it("restores LiveKit retained state when saving a new profile fails", async () => { - const publisher = new SfuPublisher(); - await publisher.connect(connection); - await publisher.activate(stream(track("video", "video-1")), qualityProfile); - const publication = livekit.state.rooms[0].localParticipant.publications[0]; - const localTrack = publication.track; - localTrack.setDegradationPreference.mockRejectedValueOnce( - new Error("preference save failed"), - ); - - await expect( - publisher.updateProfile({ - resolution: "720p", - maxFramerate: 30, - maxBitrate: 3_000_000, - degradationPreference: "maintain-framerate", - }), - ).resolves.toBe(false); - - expect(localTrack.setDegradationPreference).toHaveBeenLastCalledWith( - "maintain-resolution", - ); - expect(localTrack.savedDegradationPreference).toBe("maintain-resolution"); - expect(localTrack.publishOptions).toBe(publication.options); - expect(publication.options).toMatchObject({ - screenShareEncoding: { - maxBitrate: 8_000_000, - maxFramerate: 60, - }, - degradationPreference: "maintain-resolution", - }); - expect(publisher.getSenderParameters()?.applied).toMatchObject({ - maxBitrate: 8_000_000, - maxFramerate: 60, - degradationPreference: "maintain-resolution", - }); - expect(publisher.getQualityWarning()).toContain("应用 SFU 发送参数失败"); - expect(publisher.getQualityWarning()).not.toContain("preference save failed"); - }); - - it("does not retain an update from a disconnected publisher generation", async () => { - const publisher = new SfuPublisher(); - await publisher.connect(connection); - await publisher.activate(stream(track("video", "video-1")), qualityProfile); - const localTrack = livekit.state.rooms[0].localParticipant.publications[0].track; - const publication = livekit.state.rooms[0].localParticipant.publications[0]; - const gate = deferred(); - localTrack.setDegradationPreference.mockImplementationOnce( - async (preference) => { - await gate.promise; - localTrack.savedDegradationPreference = preference; - }, - ); - - const updating = publisher.updateProfile({ - resolution: "720p", - maxFramerate: 30, - maxBitrate: 3_000_000, - degradationPreference: "maintain-framerate", - }); - await vi.waitFor(() => - expect(localTrack.setDegradationPreference).toHaveBeenCalledTimes(2), - ); - await publisher.disconnect(); - gate.resolve(); - - await expect(updating).resolves.toBe(false); - expect(publisher.getSenderParameters()).toBeNull(); - expect(publisher.getQualityWarning()).toBeNull(); - expect(localTrack.savedDegradationPreference).toBe("maintain-resolution"); - expect(localTrack.sender.setParameters).toHaveBeenCalledTimes(1); - expect(publication.options).toMatchObject({ - screenShareEncoding: { - maxBitrate: 8_000_000, - maxFramerate: 60, - }, - degradationPreference: "maintain-resolution", - }); - }); - - it("retains a visible warning when the SFU sender rewrites a parameter", async () => { - const publisher = new SfuPublisher(); - await publisher.connect(connection); - await publisher.activate(stream(track("video", "video-1")), qualityProfile); - const sender = livekit.state.rooms[0].localParticipant.publications[0].track - .sender; - sender.setParameters.mockImplementationOnce(async (parameters) => { - sender.parameters = { - ...parameters, - encodings: parameters.encodings.map((encoding) => ({ - ...encoding, - maxBitrate: 2_000_000, - })), - }; - }); - - await expect( - publisher.updateProfile({ - resolution: "1080p", - maxFramerate: 30, - maxBitrate: 5_000_000, - degradationPreference: "balanced", - }), - ).resolves.toBe(true); - - expect(publisher.getQualityWarning()).toContain("码率上限"); - }); - - it("retains a visible warning when SFU sender parameters are rejected", async () => { - const publisher = new SfuPublisher(); - await publisher.connect(connection); - await publisher.activate(stream(track("video", "video-1")), qualityProfile); - const publication = livekit.state.rooms[0].localParticipant.publications[0]; - const localTrack = publication.track; - const sender = localTrack.sender; - sender.setParameters.mockRejectedValueOnce(new Error("unsupported")); - - await expect( - publisher.updateProfile({ - resolution: "720p", - maxFramerate: 30, - maxBitrate: 3_000_000, - degradationPreference: "maintain-framerate", - }), - ).resolves.toBe(false); - - expect(localTrack.setDegradationPreference).toHaveBeenLastCalledWith( - "maintain-resolution", - ); - expect(localTrack.savedDegradationPreference).toBe("maintain-resolution"); - expect(publication.options).toMatchObject({ - screenShareEncoding: { - maxBitrate: 8_000_000, - maxFramerate: 60, - }, - degradationPreference: "maintain-resolution", - }); - expect(sender.parameters).toMatchObject({ - degradationPreference: "maintain-resolution", - encodings: [ - expect.objectContaining({ - maxBitrate: 8_000_000, - maxFramerate: 60, - }), - ], - }); - expect(publisher.getQualityWarning()).toBe("应用 SFU 发送参数失败"); - expect(publisher.getQualityWarning()).not.toContain("unsupported"); - }); - - it("retains a sender rewrite warning after rolling back parameters", async () => { - const publisher = new SfuPublisher(); - await publisher.connect(connection); - await publisher.activate(stream(track("video", "video-1")), qualityProfile); - const sender = livekit.state.rooms[0].localParticipant.publications[0].track - .sender; - sender.setParameters - .mockRejectedValueOnce(new Error("unsupported")) - .mockImplementationOnce(async (parameters) => { - sender.parameters = { - ...parameters, - encodings: parameters.encodings.map((encoding) => ({ - ...encoding, - maxBitrate: 1_500_000, - })), - }; - }); - - await expect( - publisher.updateProfile({ - resolution: "720p", - maxFramerate: 30, - maxBitrate: 3_000_000, - degradationPreference: "balanced", - }), - ).resolves.toBe(false); - - expect(publisher.getSenderParameters()).toMatchObject({ - requested: { maxBitrate: 8_000_000 }, - applied: { maxBitrate: 1_500_000 }, - mismatches: ["maxBitrate"], - }); - expect(publisher.getQualityWarning()).toContain("应用 SFU 发送参数失败"); - expect(publisher.getQualityWarning()).not.toContain("unsupported"); - expect(publisher.getQualityWarning()).toContain("码率上限"); - }); - - it("restores the previous video after a partially applied replacement fails", async () => { - const previousVideo = track("video", "video-1"); - const nextVideo = track("video", "video-2"); - const publisher = new SfuPublisher(); - await publisher.connect(connection); - await publisher.activate(stream(previousVideo), qualityProfile); - const localTrack = livekit.state.rooms[0].localParticipant.publications[0].track; - const previousPublishedVideo = localTrack.currentTrack; - localTrack.replaceTrack.mockImplementationOnce(async (replacement) => { - localTrack.currentTrack = replacement; - throw new Error("sender swap failed after applying"); - }); - - await expect(publisher.replaceStream(stream(nextVideo))).resolves.toBe(false); - - const nextPublishedVideo = localTrack.replaceTrack.mock.calls[0]?.[0]; - expect(nextPublishedVideo).not.toBe(nextVideo); - expect(nextPublishedVideo?.id).toBe(nextVideo.id); - expect(localTrack.replaceTrack).toHaveBeenNthCalledWith( - 2, - previousPublishedVideo, - ); - expect(localTrack.currentTrack).toBe(previousPublishedVideo); - expect(nextPublishedVideo?.stop).toHaveBeenCalledOnce(); - expect(previousPublishedVideo.stop).not.toHaveBeenCalled(); - }); - - it("applies pause authority to an SFU source replacement while it is in flight", async () => { - const publisher = new SfuPublisher(); - const previousAudio = track("audio", "audio-1"); - const nextVideo = track("video", "video-2"); - const nextAudio = track("audio", "audio-2"); - await publisher.connect(connection); - await publisher.activate( - stream(track("video", "video-1"), previousAudio), - qualityProfile, - ); - const room = livekit.state.rooms[0]; - const videoPublication = room.localParticipant.publications[0].track; - const replaceGate = deferred(); - videoPublication.replaceTrack.mockImplementationOnce( - async (replacement: MediaStreamTrack) => { - await replaceGate.promise; - videoPublication.currentTrack = replacement; - videoPublication.sender.track = replacement; - }, - ); - - const replacing = publisher.replaceStream(stream(nextVideo, nextAudio)); - await vi.waitFor(() => - expect(videoPublication.replaceTrack).toHaveBeenCalledOnce(), - ); - const replacementVideo = videoPublication.replaceTrack.mock.calls[0]![0]!; - publisher.setPaused(true); - expect(replacementVideo.enabled).toBe(false); - expect(nextAudio.enabled).toBe(false); - - replaceGate.resolve(); - await expect(replacing).resolves.toBe(true); - expect(videoPublication.currentTrack.enabled).toBe(false); - expect(room.localParticipant.publications[1].track.currentTrack.enabled).toBe( - false, - ); - }); - - it("uses the screen audio preset when a replacement adds audio", async () => { - const publisher = new SfuPublisher(); - const audio = track("audio", "audio-2"); - await publisher.connect(connection); - await publisher.activate(stream(track("video", "video-1")), { - ...qualityProfile, - screenAudioQuality: "very-high", - }); - const room = livekit.state.rooms[0]; - - await expect( - publisher.replaceStream(stream(track("video", "video-2"), audio)), - ).resolves.toBe(true); - - expect(room.localParticipant.publishTrack).toHaveBeenNthCalledWith(2, audio, { - source: Track.Source.ScreenShareAudio, - audioPreset: { maxBitrate: 192_000 }, - forceStereo: true, - dtx: false, - red: false, - }); - }); - - it("reapplies and retains the current sender settings after replacing video", async () => { - const publisher = new SfuPublisher(); - await publisher.connect(connection); - await publisher.activate(stream(track("video", "video-1")), qualityProfile); - const localTrack = livekit.state.rooms[0].localParticipant.publications[0].track; - - await expect( - publisher.replaceStream(stream(track("video", "video-2"))), - ).resolves.toBe(true); - - expect(localTrack.sender.setParameters).toHaveBeenCalledTimes(2); - expect(publisher.getSenderParameters()?.requested).toMatchObject({ - maxBitrate: 8_000_000, - maxFramerate: 60, - degradationPreference: "maintain-resolution", - }); - }); - - it("restores sender settings when replacement configuration is rejected", async () => { - const previousVideo = track("video", "video-1"); - const publisher = new SfuPublisher(); - await publisher.connect(connection); - await publisher.activate(stream(previousVideo), qualityProfile); - const localTrack = livekit.state.rooms[0].localParticipant.publications[0].track; - const previousPublishedVideo = localTrack.currentTrack; - localTrack.sender.setParameters.mockRejectedValueOnce( - new Error("replacement parameters rejected"), - ); - - await expect( - publisher.replaceStream(stream(track("video", "video-2"))), - ).resolves.toBe(false); - - expect(localTrack.replaceTrack).toHaveBeenCalledTimes(2); - expect(localTrack.currentTrack).toBe(previousPublishedVideo); - expect(localTrack.sender.setParameters).toHaveBeenCalledTimes(3); - expect(publisher.getSenderParameters()?.applied.maxBitrate).toBe(8_000_000); - expect(publisher.getQualityWarning()).toBe("切换 SFU 分享来源失败"); - expect(publisher.getQualityWarning()).not.toContain( - "replacement parameters rejected", - ); - }); - - it("disconnects fail-closed when replacement rollback also fails", async () => { - const disconnected = vi.fn(); - const previousVideo = track("video", "video-1"); - const publisher = new SfuPublisher({ onDisconnected: disconnected }); - await publisher.connect(connection); - await publisher.activate(stream(previousVideo), qualityProfile); - const room = livekit.state.rooms[0]; - const localTrack = room.localParticipant.publications[0].track; - localTrack.replaceTrack - .mockRejectedValueOnce(new Error("replace failed")) - .mockRejectedValueOnce(new Error("rollback failed")); - - await expect( - publisher.replaceStream(stream(track("video", "video-2"))), - ).rejects.toThrow("rollback failed"); - expect(room.disconnect).toHaveBeenCalledWith(false); - expect(disconnected).toHaveBeenCalledOnce(); - }); - - it("does not resurrect a stale connection after disconnect", async () => { - const gate = deferred(); - livekit.state.connectGate = gate.promise; - const publisher = new SfuPublisher(); - const connecting = publisher.connect(connection); - await vi.waitFor(() => expect(livekit.state.rooms).toHaveLength(1)); - const room = livekit.state.rooms[0]; - - await publisher.disconnect(); - gate.resolve(); - - await expect(connecting).resolves.toBe(false); - expect(room.localParticipant.publishTrack).not.toHaveBeenCalled(); - expect(room.disconnect).toHaveBeenCalledOnce(); - }); -}); - -describe("SfuSubscriber", () => { - it("uses cumulative frames for a fresh exact subscriber and growth after rearm", async () => { - vi.useFakeTimers(); - let framesDecoded = 1; - const proofs: number[] = []; - const subscriber = new SfuSubscriber({ - onStream: vi.fn(), - onFirstDecodedFrame: () => { - proofs.push(framesDecoded); - return true; - }, - }); - await subscriber.connect(connection); - const room = livekit.state.rooms[0]; - expect(room.options).toEqual({ disconnectOnPageLeave: false }); - const host = new livekit.FakeRemoteParticipant("host"); - const publication = new livekit.FakeRemotePublication( - "host-video", - Track.Source.ScreenShare, - ); - host.add(publication); - room.remoteParticipants.set("host", host); - expect(subscriber.activate()).toBe(true); - subscriber.armDecodedFrameProof(); - const video = track("video", "video-1"); - room.emit( - RoomEvent.TrackSubscribed, - remoteTrack(video, () => - statsReport([ - { - id: "video-in", - type: "inbound-rtp", - timestamp: 1_000, - kind: "video", - framesDecoded, - }, - ]), - ), - publication, - host, - ); - await vi.waitFor(() => expect(proofs).toEqual([1])); - - proofs.length = 0; - framesDecoded = 7; - subscriber.armDecodedFrameProof(true); - await Promise.resolve(); - await vi.advanceTimersByTimeAsync(100); - expect(proofs).toEqual([]); - framesDecoded = 8; - await vi.advanceTimersByTimeAsync(300); - expect(proofs).toEqual([8]); - - subscriber.deactivate(); - framesDecoded = 9; - await vi.advanceTimersByTimeAsync(500); - expect(proofs).toEqual([8]); - }); - - it("retains pending frame proof through LiveKit full reconnect order", async () => { - vi.useFakeTimers(); - const proofs = vi.fn(() => true); - const subscriber = new SfuSubscriber({ - onStream: vi.fn(), - onFirstDecodedFrame: proofs, - }); - await subscriber.connect(connection); - const room = livekit.state.rooms[0]; - const oldHost = new livekit.FakeRemoteParticipant("host"); - const oldPublication = new livekit.FakeRemotePublication( - "host-video-before-reconnect", - Track.Source.ScreenShare, - ); - oldHost.add(oldPublication); - room.remoteParticipants.set("host", oldHost); - expect(subscriber.activate()).toBe(true); - subscriber.armDecodedFrameProof(); - - let releaseOldRead!: () => void; - const oldRead = new Promise((resolve) => { - releaseOldRead = resolve; - }); - room.emit( - RoomEvent.TrackSubscribed, - remoteTrack(track("video", "video-before-reconnect"), async () => { - await oldRead; - return statsReport([]); - }), - oldPublication, - oldHost, - ); - - room.remoteParticipants.delete("host"); - room.emit(RoomEvent.ParticipantDisconnected, oldHost); - room.emit(RoomEvent.Reconnecting); - - const newHost = new livekit.FakeRemoteParticipant("host"); - const newPublication = new livekit.FakeRemotePublication( - "host-video-after-reconnect", - Track.Source.ScreenShare, - ); - newHost.add(newPublication); - room.remoteParticipants.set("host", newHost); - room.emit(RoomEvent.Reconnected); - room.emit( - RoomEvent.TrackSubscribed, - remoteTrack(track("video", "video-after-reconnect"), async () => - statsReport([ - { - id: "video-after-reconnect", - type: "inbound-rtp", - timestamp: 1, - kind: "video", - framesDecoded: 1, - }, - ]), - ), - newPublication, - newHost, - ); - - await vi.waitFor(() => expect(proofs).toHaveBeenCalledOnce()); - releaseOldRead(); - expect(subscriber.deactivate()).toBe(true); - }); - - it("reconciles a Host publication announced while connect is pending", async () => { - const gate = deferred(); - livekit.state.connectGate = gate.promise; - const subscriber = new SfuSubscriber({ onStream: vi.fn() }); - const connecting = subscriber.connect(connection); - await vi.waitFor(() => expect(livekit.state.rooms).toHaveLength(1)); - const room = livekit.state.rooms[0]; - const host = new livekit.FakeRemoteParticipant("host"); - const hostVideo = new livekit.FakeRemotePublication( - "host-video-pending", - Track.Source.ScreenShare, - ); - host.add(hostVideo); - - room.emit(RoomEvent.TrackPublished, hostVideo, host); - expect(hostVideo.setSubscribed).not.toHaveBeenCalled(); - gate.resolve(); - await expect(connecting).resolves.toBe(true); - - expect(subscriber.activate()).toBe(true); - expect(hostVideo.setSubscribed).toHaveBeenCalledWith(true); - }); - - it("reconciles a Host participant that appears after activation", async () => { - const subscriber = new SfuSubscriber({ onStream: vi.fn() }); - await subscriber.connect(connection); - const room = livekit.state.rooms[0]; - expect(subscriber.activate()).toBe(true); - - const host = new livekit.FakeRemoteParticipant("host"); - const hostVideo = new livekit.FakeRemotePublication( - "host-video-late", - Track.Source.ScreenShare, - ); - host.add(hostVideo); - room.emit(RoomEvent.ParticipantConnected, host); - - expect(hostVideo.setSubscribed).toHaveBeenCalledWith(true); - }); - - it("subscribes only to assigned Host screen tracks", async () => { - const gate = deferred(); - livekit.state.connectGate = gate.promise; - const streams: Array = []; - const subscriber = new SfuSubscriber({ - onStream: (nextStream) => streams.push(nextStream), - }); - const connecting = subscriber.connect(connection); - await vi.waitFor(() => expect(livekit.state.rooms).toHaveLength(1)); - const room = livekit.state.rooms[0]; - const host = new livekit.FakeRemoteParticipant("host"); - const hostVideo = new livekit.FakeRemotePublication( - "host-video", - Track.Source.ScreenShare, - ); - const hostAudio = new livekit.FakeRemotePublication( - "host-audio", - Track.Source.ScreenShareAudio, - ); - const hostCamera = new livekit.FakeRemotePublication( - "host-camera", - Track.Source.Camera, - ); - host.add(hostVideo).add(hostAudio).add(hostCamera); - room.remoteParticipants.set("host", host); - const viewer = new livekit.FakeRemoteParticipant("viewer:other"); - const viewerScreen = new livekit.FakeRemotePublication( - "viewer-screen", - Track.Source.ScreenShare, - ); - viewer.add(viewerScreen); - room.remoteParticipants.set(viewer.identity, viewer); - - gate.resolve(); - await expect(connecting).resolves.toBe(true); - expect(room.connect).toHaveBeenCalledWith(connection.url, connection.token, { - autoSubscribe: false, - rtcConfig: { iceServers: [] }, - }); - expect(hostVideo.setSubscribed).not.toHaveBeenCalled(); - expect(streams).toEqual([]); - - expect(subscriber.activate()).toBe(true); - expect(hostVideo.setSubscribed).toHaveBeenCalledWith(true); - expect(hostAudio.setSubscribed).toHaveBeenCalledWith(true); - expect(hostCamera.setSubscribed).not.toHaveBeenCalled(); - expect(viewerScreen.setSubscribed).not.toHaveBeenCalled(); - - const audio = track("audio", "audio-1"); - room.emit( - RoomEvent.TrackSubscribed, - remoteTrack(audio), - hostAudio, - host, - ); - expect(streams).toEqual([]); - - const video = track("video", "video-1"); - room.emit( - RoomEvent.TrackSubscribed, - remoteTrack(video), - hostVideo, - host, - ); - expect(streams.at(-1)?.getVideoTracks()).toEqual([video]); - expect(streams.at(-1)?.getAudioTracks()).toEqual([audio]); - - expect(subscriber.deactivate()).toBe(true); - expect(hostVideo.setSubscribed).toHaveBeenLastCalledWith(false); - expect(hostAudio.setSubscribed).toHaveBeenLastCalledWith(false); - expect(streams.at(-1)).toBeNull(); - expect(room.disconnect).not.toHaveBeenCalled(); - }); - - it("keeps one stream while video-first tracks change", async () => { - const streams: Array = []; - const availability: boolean[] = []; - const subscriber = new SfuSubscriber({ - onStream: (nextStream) => streams.push(nextStream), - onVideoAvailability: (available) => availability.push(available), - }); - await subscriber.connect(connection); - const room = livekit.state.rooms[0]; - const host = new livekit.FakeRemoteParticipant("host"); - const hostVideo = new livekit.FakeRemotePublication( - "host-video", - Track.Source.ScreenShare, - ); - const hostAudio = new livekit.FakeRemotePublication( - "host-audio", - Track.Source.ScreenShareAudio, - ); - host.add(hostVideo).add(hostAudio); - room.remoteParticipants.set("host", host); - expect(subscriber.activate()).toBe(true); - const video = track("video", "video-1"); - room.emit(RoomEvent.TrackSubscribed, remoteTrack(video), hostVideo, host); - const stableStream = streams.at(-1); - expect(stableStream?.getTracks()).toEqual([video]); - expect(availability).toEqual([true]); - const audio = track("audio", "audio-1"); - room.emit(RoomEvent.TrackSubscribed, remoteTrack(audio), hostAudio, host); - expect(streams.at(-1)).toBe(stableStream); - expect(stableStream?.getAudioTracks()).toEqual([audio]); - const callbacksBeforeReplacement = streams.length; - const replacement = track("video", "video-2"); - room.emit(RoomEvent.TrackSubscribed, remoteTrack(replacement), hostVideo, host); - expect(streams).toHaveLength(callbacksBeforeReplacement + 1); - expect(streams.at(-1)).toBe(stableStream); - expect(stableStream?.getVideoTracks()).toEqual([replacement]); - video.dispatchEvent(new Event("ended")); - expect(stableStream?.getVideoTracks()).toEqual([replacement]); - expect(availability).toEqual([true]); - replacement.dispatchEvent(new Event("ended")); - expect(stableStream?.getVideoTracks()).toEqual([]); - expect(streams).toHaveLength(callbacksBeforeReplacement + 1); - expect(availability).toEqual([true, false]); - - const recovered = track("video", "video-3"); - room.emit(RoomEvent.TrackSubscribed, remoteTrack(recovered), hostVideo, host); - expect(streams.at(-1)).toBe(stableStream); - expect(stableStream?.getVideoTracks()).toEqual([recovered]); - expect(availability).toEqual([true, false, true]); - }); - - it.each(["unsubscribed", "unpublished", "host-disconnected"] as const)( - "reports the last video unavailable when it is %s", - async (loss) => { - const availability: boolean[] = []; - const subscriber = new SfuSubscriber({ - onStream: vi.fn(), - onVideoAvailability: (available) => availability.push(available), - }); - await subscriber.connect(connection); - const room = livekit.state.rooms[0]; - const host = new livekit.FakeRemoteParticipant("host"); - const publication = new livekit.FakeRemotePublication( - "host-video", - Track.Source.ScreenShare, - ); - host.add(publication); - room.remoteParticipants.set("host", host); - subscriber.activate(); - const video = track("video", "video-1"); - const remoteVideo = remoteTrack(video); - room.emit(RoomEvent.TrackSubscribed, remoteVideo, publication, host); - availability.length = 0; - - if (loss === "unsubscribed") { - room.emit(RoomEvent.TrackUnsubscribed, remoteVideo, publication, host); - } else if (loss === "unpublished") { - room.emit(RoomEvent.TrackUnpublished, publication, host); - } else { - room.emit(RoomEvent.ParticipantDisconnected, host); - } - - expect(availability).toEqual([false]); - }, - ); - - it("keeps active decoded-frame sampling through unavailable receiver stats", async () => { - vi.useFakeTimers(); - const samples: Array = []; - const updates: ConnectionMetrics[] = []; - const subscriber = new SfuSubscriber({ - onStream: vi.fn(), - onStats: (metrics) => updates.push(metrics), - onDecodedFrameSample: (framesDecodedDelta) => - samples.push(framesDecodedDelta), - }); - await subscriber.connect(connection); - const room = livekit.state.rooms[0]; - const host = new livekit.FakeRemoteParticipant("host"); - const publication = new livekit.FakeRemotePublication( - "host-video", - Track.Source.ScreenShare, - ); - host.add(publication); - room.remoteParticipants.set("host", host); - - expect(subscriber.activate()).toBe(true); - expect(samples).toEqual([null]); - - let receiverStats: RTCStatsReport | Promise | Error = - statsReport([]); - const video = track("video", "video-1"); - const remoteVideo = remoteTrack(video, () => { - if (receiverStats instanceof Error) throw receiverStats; - return receiverStats; - }); - room.emit(RoomEvent.TrackSubscribed, remoteVideo, publication, host); - await vi.advanceTimersByTimeAsync(0); - expect(samples).toEqual([null, null]); - expect(updates).toEqual([]); - - receiverStats = new Error("stats unavailable"); - const samplesBeforeStatsError = samples.length; - await vi.advanceTimersByTimeAsync(2_000); - expect(samples).toHaveLength(samplesBeforeStatsError + 1); - expect(samples.at(-1)).toBeNull(); - expect(updates).toEqual([]); - - receiverStats = statsReport([ - { - id: "video-in", - type: "inbound-rtp", - timestamp: 1_000, - kind: "video", - trackIdentifier: video.id, - framesDecoded: 1, - }, + expect(send.mock.calls[1]![0].kind).toBe("candidate"); + expect(clientMessageSchema.safeParse(offer).success).toBe(true); + expect(pc.transceivers[0]!.setCodecPreferences.mock.calls[0]![0]).toEqual([ + expect.objectContaining({ mimeType: "video/H264" }), ]); - await vi.advanceTimersByTimeAsync(2_000); - expect(samples.at(-1)).toBeNull(); - expect(updates).toHaveLength(1); - - receiverStats = statsReport([ - { - id: "video-in", - type: "inbound-rtp", - timestamp: 3_000, - kind: "video", - trackIdentifier: video.id, - framesDecoded: 5, - }, - ]); - await vi.advanceTimersByTimeAsync(2_000); - expect(samples.at(-1)).toBe(4); - expect(updates).toHaveLength(2); - - let releasePendingStats!: (report: RTCStatsReport) => void; - receiverStats = new Promise((resolve) => { - releasePendingStats = resolve; - }); - const statsCallsBeforePending = remoteVideo.getRTCStatsReport.mock.calls.length; - const samplesBeforePending = samples.length; - await vi.advanceTimersByTimeAsync(2_000); - expect(remoteVideo.getRTCStatsReport).toHaveBeenCalledTimes( - statsCallsBeforePending + 1, + await host.acceptSignal( + signal({ description: { type: "answer", sdp: "answer" } }), ); - expect(samples).toHaveLength(samplesBeforePending); - await vi.advanceTimersByTimeAsync(2_000); - expect(remoteVideo.getRTCStatsReport).toHaveBeenCalledTimes( - statsCallsBeforePending + 1, - ); - expect(samples).toHaveLength(samplesBeforePending + 1); - expect(samples.at(-1)).toBeNull(); - - const replacementVideo = track("video", "video-2"); - let replacementStats = statsReport([ - { - id: "replacement-video-in", - type: "inbound-rtp", - timestamp: 1_000, - kind: "video", - trackIdentifier: replacementVideo.id, - framesDecoded: 2, - }, - ]); - const replacementRemoteVideo = remoteTrack( - replacementVideo, - () => replacementStats, - ); - room.emit( - RoomEvent.TrackSubscribed, - replacementRemoteVideo, - publication, - host, - ); - await vi.advanceTimersByTimeAsync(0); - expect(updates.at(-1)?.trackIdentifier).toBe(replacementVideo.id); - const samplesAfterReplacement = samples.length; - const updatesAfterReplacement = updates.length; - releasePendingStats( - statsReport([ - { - id: "video-in", - type: "inbound-rtp", - timestamp: 5_000, - kind: "video", - trackIdentifier: video.id, - framesDecoded: 9, - }, - ]), - ); - await Promise.resolve(); - await Promise.resolve(); - expect(samples).toHaveLength(samplesAfterReplacement); - expect(updates).toHaveLength(updatesAfterReplacement); - - replacementStats = statsReport([ - { - id: "replacement-video-in", - type: "inbound-rtp", - timestamp: 3_000, - kind: "video", - trackIdentifier: replacementVideo.id, - framesDecoded: 7, - }, - ]); - await vi.advanceTimersByTimeAsync(2_000); - expect(samples.at(-1)).toBe(5); - - room.emit( - RoomEvent.TrackUnsubscribed, - replacementRemoteVideo, - publication, - host, - ); - const samplesBeforeMissingTrack = samples.length; - await vi.advanceTimersByTimeAsync(2_000); - expect(samples).toHaveLength(samplesBeforeMissingTrack + 1); - expect(samples.at(-1)).toBeNull(); - - expect(subscriber.deactivate()).toBe(true); - const samplesAfterDeactivate = samples.length; - await vi.advanceTimersByTimeAsync(4_000); - expect(samples).toHaveLength(samplesAfterDeactivate); + expect(pc.remoteDescription?.type).toBe("answer"); }); - it("keeps video liveness independent from optional audio stats", async () => { - vi.useFakeTimers(); - const samples: Array = []; - const updates: ConnectionMetrics[] = []; - const subscriber = new SfuSubscriber({ - onStream: vi.fn(), - onStats: (metrics) => updates.push(metrics), - onDecodedFrameSample: (framesDecodedDelta) => - samples.push(framesDecodedDelta), - }); - await subscriber.connect(connection); - const room = livekit.state.rooms[0]; - const host = new livekit.FakeRemoteParticipant("host"); - const hostVideo = new livekit.FakeRemotePublication( - "host-video", - Track.Source.ScreenShare, - ); - const hostAudio = new livekit.FakeRemotePublication( - "host-audio", - Track.Source.ScreenShareAudio, - ); - host.add(hostVideo).add(hostAudio); - room.remoteParticipants.set("host", host); - expect(subscriber.activate()).toBe(true); - - const video = track("video", "video-1"); - let videoTimestamp = 1_000; - let videoFrames = 1; - const remoteVideo = remoteTrack(video, () => - statsReport([ - { - id: "video-in", - type: "inbound-rtp", - timestamp: videoTimestamp, - kind: "video", - trackIdentifier: video.id, - framesDecoded: videoFrames, - estimatedPlayoutTimestamp: 10_000, - }, - ]), - ); - const audio = track("audio", "audio-1"); - let audioStats: RTCStatsReport | Promise | Error = - new Error("audio stats unavailable"); - const remoteAudio = remoteTrack(audio, () => { - if (audioStats instanceof Error) throw audioStats; - return audioStats; - }); - room.emit(RoomEvent.TrackSubscribed, remoteAudio, hostAudio, host); - room.emit(RoomEvent.TrackSubscribed, remoteVideo, hostVideo, host); - await vi.advanceTimersByTimeAsync(0); - expect(samples.at(-1)).toBeNull(); - expect(updates).toHaveLength(1); - - videoTimestamp = 3_000; - videoFrames = 5; - await vi.advanceTimersByTimeAsync(2_000); - expect(samples.at(-1)).toBe(4); - expect(remoteAudio.getRTCStatsReport).toHaveBeenCalledTimes(2); - - audioStats = receiverReport("audio"); - videoTimestamp = 5_000; - videoFrames = 9; - await vi.advanceTimersByTimeAsync(2_000); - expect(samples.at(-1)).toBe(4); - expect(updates.at(-1)?.audioVideoPlayoutDeltaMs).toBe(12); - expect(remoteAudio.getRTCStatsReport).toHaveBeenCalledTimes(3); - - audioStats = new Promise(() => undefined); - videoTimestamp = 7_000; - videoFrames = 13; - const samplesBeforePendingAudio = samples.length; - await vi.advanceTimersByTimeAsync(2_000); - expect(samples).toHaveLength(samplesBeforePendingAudio + 1); - expect(samples.at(-1)).toBe(4); - expect(updates.at(-1)?.audioVideoPlayoutDeltaMs).toBeNull(); - expect(remoteAudio.getRTCStatsReport).toHaveBeenCalledTimes(4); - - videoTimestamp = 9_000; - videoFrames = 17; - const samplesBeforeSecondPendingAudioTick = samples.length; - await vi.advanceTimersByTimeAsync(2_000); - expect(samples).toHaveLength(samplesBeforeSecondPendingAudioTick + 1); - expect(samples.at(-1)).toBe(4); - expect(updates.at(-1)?.audioVideoPlayoutDeltaMs).toBeNull(); - expect(remoteAudio.getRTCStatsReport).toHaveBeenCalledTimes(4); - expect(remoteVideo.getRTCStatsReport).toHaveBeenCalledTimes(5); - - expect(subscriber.deactivate()).toBe(true); - }); - - it("reports merged receiver stats and drops a stale sample", async () => { - const updates: ConnectionMetrics[] = []; - const samples: Array = []; - const states: string[] = []; - const subscriber = new SfuSubscriber({ - onStream: vi.fn(), - onStats: (metrics) => updates.push(metrics), - onDecodedFrameSample: (framesDecodedDelta) => - samples.push(framesDecodedDelta), - onState: (state) => states.push(state), - }); - await subscriber.connect(connection); - const room = livekit.state.rooms[0]; - const host = new livekit.FakeRemoteParticipant("host"); - const hostVideo = new livekit.FakeRemotePublication( - "host-video", - Track.Source.ScreenShare, - ); - const hostAudio = new livekit.FakeRemotePublication( - "host-audio", - Track.Source.ScreenShareAudio, - ); - host.add(hostVideo).add(hostAudio); - room.remoteParticipants.set("host", host); - expect(subscriber.activate()).toBe(true); - - const reports = { - video: receiverReport("video"), - audio: receiverReport("audio"), + it("queues remote ICE and rejects retired connection/publication identities", async () => { + const { publisher: host, pc } = await publisher(); + const candidate = { + candidate: "candidate:1 1 UDP 1 127.0.0.1 4100 typ host", }; - expect( - Array.from(mergeStatsReports([reports.video, reports.audio])!.keys()), - ).toEqual(["video-in", "audio-in"]); - const video = track("video", "video-1"); - const audio = track("audio", "audio-1"); - let videoStats: RTCStatsReport | Promise = reports.video; - const remoteVideo = remoteTrack(video, () => videoStats); - room.emit( - RoomEvent.TrackSubscribed, - remoteTrack(audio, () => reports.audio), - hostAudio, - host, + await host.acceptSignal(signal({ kind: "candidate", candidate })); + expect(pc.addIceCandidate).not.toHaveBeenCalled(); + await host.acceptSignal( + signal({ + connectionId: "retired_12345678", + description: { type: "answer", sdp: "stale" }, + }), ); - room.emit( - RoomEvent.TrackSubscribed, - remoteVideo, - hostVideo, - host, + await host.acceptSignal( + signal({ + publicationGeneration: "retired_12345678", + description: { type: "answer", sdp: "stale" }, + }), ); - await vi.waitFor(() => expect(updates).toHaveLength(1)); - expect(samples).toEqual([null, null]); - - expect(updates.at(-1)).toMatchObject({ - rtpStatsId: "video-in", - trackIdentifier: "video-1", - audioVideoPlayoutDeltaMs: 12, - }); - - let releaseStats = (): void => undefined; - videoStats = new Promise((resolve) => { - releaseStats = () => resolve(reports.video); - }); - room.emit(RoomEvent.Reconnecting); - room.emit(RoomEvent.Reconnected); - expect(states).toEqual(["reconnecting", "connected"]); - await vi.waitFor(() => - expect(remoteVideo.getRTCStatsReport).toHaveBeenCalledTimes(2), + expect(pc.setRemoteDescription).not.toHaveBeenCalled(); + await host.acceptSignal( + signal({ description: { type: "answer", sdp: "answer" } }), ); - - expect(subscriber.deactivate()).toBe(true); - const samplesAfterDeactivate = samples.length; - releaseStats(); - await Promise.resolve(); - await Promise.resolve(); - expect(updates).toHaveLength(1); - expect(samples).toHaveLength(samplesAfterDeactivate); + expect(pc.addIceCandidate).toHaveBeenCalledWith(candidate); }); - it("notifies the controller after a terminal room disconnect", async () => { - const disconnected = vi.fn(); + it("updates all existing encodings and audio without changing the PC or codec", async () => { + const { publisher: host, pc, send } = await publisher(); + const profile = { + ...QUALITY_PROFILES["720p30"], + screenAudioQuality: "very-high" as const, + }; + expect(await host.updateProfile(profile)).toBe(true); + expect(pc.transceivers[0]!.sender.parameters.encodings).toEqual([ + expect.objectContaining({ + rid: "q", + maxBitrate: 750_000, + scaleResolutionDownBy: 3, + }), + expect.objectContaining({ + rid: "h", + maxBitrate: 3_000_000, + scaleResolutionDownBy: 1.5, + }), + ]); + expect(host.getAudioSenderParameters()?.appliedMaxBitrate).toBe(192_000); + expect(send.mock.calls.at(-1)![0]).toMatchObject({ + kind: "media", + media: { codec: "h264", audioBitrate: 192_000 }, + }); + expect(FakePc.instances).toHaveLength(1); + expect(pc.createOffer).toHaveBeenCalledOnce(); + }); + + it("reuses the audio transceiver when source audio appears and retires clones only", async () => { + const { publisher: host, pc, video, send } = await publisher(false); + const nextVideo = new FakeTrack("video"); + const nextAudio = new FakeTrack("audio"); + host.setPaused(true); + expect(await host.replaceStream(stream(nextVideo, nextAudio))).toBe(true); + expect(video.clones[0]!.stop).toHaveBeenCalledOnce(); + expect(video.stop).not.toHaveBeenCalled(); + expect(pc.transceivers).toHaveLength(2); + expect(nextVideo.clones[0]!.enabled).toBe(false); + expect(nextAudio.clones[0]!.enabled).toBe(false); + expect(send.mock.calls.at(-1)![0]).toMatchObject({ + kind: "media", + media: { audio: true }, + }); + await host.disconnect(); + expect(nextVideo.clones[0]!.stop).toHaveBeenCalledOnce(); + expect(nextAudio.clones[0]!.stop).toHaveBeenCalledOnce(); + expect(nextVideo.stop).not.toHaveBeenCalled(); + expect(nextAudio.stop).not.toHaveBeenCalled(); + expect(pc.close).toHaveBeenCalledOnce(); + }); + + it("rolls back a failed source replacement and releases its unused clone", async () => { + const { publisher: host, pc, video } = await publisher(); + pc.transceivers[0]!.sender.replaceTrack.mockRejectedValueOnce( + new Error("replace rejected"), + ); + const replacement = new FakeTrack("video"); + expect(await host.replaceStream(stream(replacement))).toBe(false); + expect(pc.transceivers[0]!.sender.track).toBe(video.clones[0]); + expect(video.clones[0]!.stop).not.toHaveBeenCalled(); + expect(replacement.clones[0]!.stop).toHaveBeenCalledOnce(); + expect(pc.close).not.toHaveBeenCalled(); + }); + + it("preserves media during a signaling outage and updates only the route fence", async () => { + const { publisher: host, pc, send, onDisconnected } = await publisher(); + await host.acceptSignal( + signal({ description: { type: "answer", sdp: "answer" } }), + ); + pc.state("connected"); + send.mockReturnValue(false); + expect(await host.updateProfile(QUALITY_PROFILES["720p30"])).toBe(false); + expect(pc.close).not.toHaveBeenCalled(); + expect(onDisconnected).not.toHaveBeenCalled(); + send.mockReturnValue(true); + host.updateConfig({ ...config, revision: 9 }); + expect(await host.updateProfile(QUALITY_PROFILES["720p30"])).toBe(true); + const updated = send.mock.calls.at(-1)![0]; + expect(updated.revision).toBe(9); + expect(clientMessageSchema.safeParse(updated).success).toBe(true); + }); + + it("answers the server offer and commits only decoded video, preserving media during ICE restart", async () => { + const send = vi.fn(() => true); + const onFirstDecodedFrame = vi.fn(() => true); + const onStream = vi.fn(); + const onDisconnected = vi.fn(); const subscriber = new SfuSubscriber({ - onStream: vi.fn(), - onDisconnected: disconnected, + send, + onStream, + onFirstDecodedFrame, + onDisconnected, }); - await subscriber.connect(connection); - const room = livekit.state.rooms[0]; - - room.emit(RoomEvent.Disconnected); - room.emit(RoomEvent.Disconnected); - - expect(disconnected).toHaveBeenCalledOnce(); - expect(subscriber.deactivate()).toBe(false); + cleanups.push(() => subscriber.disconnect()); + await subscriber.connect(config); + subscriber.activate(); + subscriber.armDecodedFrameProof(); + const pc = FakePc.instances[0]!; + await subscriber.acceptSignal( + signal({ description: { type: "offer", sdp: "offer" } }), + ); + let framesDecoded = 0; + const video = new FakeTrack("video"); + pc.ontrack?.({ + track: video, + receiver: { + getStats: async () => + new Map([ + [ + "video", + { + id: "video", + type: "inbound-rtp", + kind: "video", + framesDecoded, + }, + ], + ]), + }, + }); + pc.state("connected"); + await vi.advanceTimersByTimeAsync(100); + expect(onFirstDecodedFrame).not.toHaveBeenCalled(); + framesDecoded = 1; + await vi.advanceTimersByTimeAsync(100); + expect(onFirstDecodedFrame).toHaveBeenCalledOnce(); + expect(onStream).toHaveBeenCalledOnce(); + expect(subscriber.reconnect()).toBe(true); + expect(send).toHaveBeenLastCalledWith( + expect.objectContaining({ + kind: "subscribe", + connectionId: config.connectionId, + }), + ); + expect(pc.close).not.toHaveBeenCalled(); + expect(onDisconnected).not.toHaveBeenCalled(); + await subscriber.disconnect(); + expect(onStream).toHaveBeenLastCalledWith(null); }); - it("does not activate a stale connection after disconnect", async () => { - const gate = deferred(); - livekit.state.connectGate = gate.promise; - const subscriber = new SfuSubscriber({ onStream: vi.fn() }); - const connecting = subscriber.connect(connection); - await vi.waitFor(() => expect(livekit.state.rooms).toHaveLength(1)); - const room = livekit.state.rooms[0]; + it("bounds pending ICE and reports one terminal failure", async () => { + const { publisher: host, pc, onDisconnected } = await publisher(); + for (let index = 0; index < 65; index++) { + await host.acceptSignal( + signal({ + kind: "candidate", + candidate: { candidate: `candidate:${index}` }, + }), + ); + } + expect(pc.close).toHaveBeenCalledOnce(); + expect(onDisconnected).toHaveBeenCalledOnce(); + expect(host.getFailureStage()).toBe("transport"); + }); - await subscriber.disconnect(); - await subscriber.disconnect(); - gate.resolve(); + it("changes audio ceilings without touching video capture or sender parameters", async () => { + const { publisher: host, pc, video } = await publisher(); + const constraints = video.clones[0]!.applyConstraints.mock.calls.length; + const parameters = + pc.transceivers[0]!.sender.setParameters.mock.calls.length; + expect( + await host.updateProfile({ + ...QUALITY_PROFILES["1080p30"], + screenAudioQuality: "saver", + }), + ).toBe(true); + expect(video.clones[0]!.applyConstraints).toHaveBeenCalledTimes( + constraints, + ); + expect(pc.transceivers[0]!.sender.setParameters).toHaveBeenCalledTimes( + parameters, + ); + expect(host.getAudioSenderParameters()?.appliedMaxBitrate).toBe(64_000); + }); - await expect(connecting).resolves.toBe(false); - expect(room.disconnect).toHaveBeenCalledOnce(); - expect(() => subscriber.activate()).toThrow("expected prepared"); + it("closes every owned clone during in-flight replacement without stopping the sources", async () => { + const { publisher: host, pc, video, sound } = await publisher(); + let release!: () => void; + const blocked = new Promise((resolve) => { + release = resolve; + }); + pc.transceivers[0]!.sender.setParameters.mockImplementationOnce( + async () => blocked, + ); + const replacement = new FakeTrack("video"); + const replacing = host.replaceStream(stream(replacement)); + await vi.advanceTimersByTimeAsync(0); + host.setPaused(true); + expect(replacement.clones[0]!.enabled).toBe(false); + await host.disconnect(); + release(); + expect(await replacing).toBe(false); + for (const track of [ + video.clones[0]!, + sound.clones[0]!, + replacement.clones[0]!, + ]) + expect(track.stop).toHaveBeenCalledOnce(); + for (const source of [video, sound, replacement]) + expect(source.stop).not.toHaveBeenCalled(); + expect(pc.close).toHaveBeenCalledOnce(); + }); + + it("fails closed when restoring a replaced sender fails", async () => { + const { publisher: host, pc, onDisconnected, video } = await publisher(); + pc.transceivers[0]!.sender.replaceTrack.mockRejectedValue( + new Error("sender failed"), + ); + const replacement = new FakeTrack("video"); + expect(await host.replaceStream(stream(replacement))).toBe(false); + expect(onDisconnected).toHaveBeenCalledOnce(); + expect(pc.close).toHaveBeenCalledOnce(); + expect(video.clones[0]!.stop).toHaveBeenCalledOnce(); + expect(replacement.clones[0]!.stop).toHaveBeenCalledOnce(); + }); + + it("uses the fenced SFU demand prefix without live profiles reactivating upper layers", async () => { + const { publisher: host, pc } = await publisher(); + await host.acceptSignal(signal({ kind: "layers", activeCount: 1 })); + expect( + pc.transceivers[0]!.sender.parameters.encodings.map( + (encoding) => encoding.active, + ), + ).toEqual([true, false]); + await host.updateProfile(QUALITY_PROFILES["720p30"]); + expect( + pc.transceivers[0]!.sender.parameters.encodings.map( + (encoding) => encoding.active, + ), + ).toEqual([true, false]); + await host.acceptSignal( + signal({ + kind: "layers", + activeCount: 0, + connectionId: "retired_connection", + }), + ); + expect( + pc.transceivers[0]!.sender.parameters.encodings.map( + (encoding) => encoding.active, + ), + ).toEqual([true, false]); + await host.acceptSignal(signal({ kind: "layers", activeCount: 0 })); + expect( + pc.transceivers[0]!.sender.parameters.encodings.map( + (encoding) => encoding.active, + ), + ).toEqual([false, false]); + await host.acceptSignal(signal({ kind: "layers", activeCount: 2 })); + expect( + pc.transceivers[0]!.sender.parameters.encodings.map( + (encoding) => encoding.active, + ), + ).toEqual([true, true]); + expect(FakePc.instances).toHaveLength(1); + }); + + it("resumes an unsent ICE-restart offer after signaling returns on the same connection", async () => { + const { publisher: host, pc, send, onDisconnected } = await publisher(); + await host.acceptSignal( + signal({ description: { type: "answer", sdp: "answer" } }), + ); + send.mockReturnValue(false); + pc.state("disconnected"); + await vi.advanceTimersByTimeAsync(0); + expect(pc.createOffer).toHaveBeenLastCalledWith({ iceRestart: true }); + expect(pc.close).not.toHaveBeenCalled(); + send.mockClear(); + send.mockReturnValue(true); + host.updateConfig({ ...config, revision: 5 }); + expect(send.mock.calls.map(([message]) => message.kind)).toEqual([ + "description", + "candidate", + ]); + expect(send.mock.calls[0]![0].revision).toBe(5); + expect(pc.close).not.toHaveBeenCalled(); + expect(onDisconnected).not.toHaveBeenCalled(); }); }); diff --git a/tests/sfu-standby-prewarmer.test.ts b/tests/sfu-standby-prewarmer.test.ts deleted file mode 100644 index 9d7d40d..0000000 --- a/tests/sfu-standby-prewarmer.test.ts +++ /dev/null @@ -1,115 +0,0 @@ -import { describe, expect, it, vi } from "vitest"; - -import { SfuStandbyPrewarmer } from "../src/client/media/sfu-standby-prewarmer.ts"; - -function deferred() { - let resolve!: (value: T) => void; - const promise = new Promise((next) => { - resolve = next; - }); - return { promise, resolve }; -} - -describe("SfuStandbyPrewarmer", () => { - it("does nothing when fallback is not advertised", () => { - const load = vi.fn(); - const prewarmer = new SfuStandbyPrewarmer(load); - - prewarmer.setUrl(undefined); - prewarmer.setUrl(null); - - expect(load).not.toHaveBeenCalled(); - }); - - it("imports and warms each advertised URL at most once", async () => { - const prepareConnection = vi.fn().mockResolvedValue(undefined); - const load = vi.fn().mockResolvedValue({ - Room: class { - prepareConnection = prepareConnection; - }, - }); - const prewarmer = new SfuStandbyPrewarmer(load); - - prewarmer.setUrl("wss://sfu.example.test"); - prewarmer.setUrl("wss://sfu.example.test"); - await vi.waitFor(() => expect(prepareConnection).toHaveBeenCalledOnce()); - prewarmer.setUrl("wss://sfu.example.test"); - - expect(load).toHaveBeenCalledOnce(); - expect(prepareConnection).toHaveBeenCalledWith( - "wss://sfu.example.test", - ); - }); - - it("drops a stale authentication before the SDK import completes", async () => { - const module = deferred<{ - Room: new () => { prepareConnection(url: string): Promise }; - }>(); - const prepareConnection = vi.fn().mockResolvedValue(undefined); - const load = vi.fn(() => module.promise); - const prewarmer = new SfuStandbyPrewarmer(load); - - prewarmer.setUrl("wss://stale.example.test"); - prewarmer.setUrl(null); - module.resolve({ - Room: class { - prepareConnection = prepareConnection; - }, - }); - await module.promise; - await Promise.resolve(); - - expect(prepareConnection).not.toHaveBeenCalled(); - }); - - it("keeps only the latest advertised URL across an import race", async () => { - const module = deferred<{ - Room: new () => { prepareConnection(url: string): Promise }; - }>(); - const prepareConnection = vi.fn().mockResolvedValue(undefined); - const load = vi.fn(() => module.promise); - const prewarmer = new SfuStandbyPrewarmer(load); - - prewarmer.setUrl("wss://stale.example.test"); - prewarmer.setUrl("wss://current.example.test"); - module.resolve({ - Room: class { - prepareConnection = prepareConnection; - }, - }); - await vi.waitFor(() => expect(prepareConnection).toHaveBeenCalledOnce()); - - expect(load).toHaveBeenCalledOnce(); - expect(prepareConnection).toHaveBeenCalledWith( - "wss://current.example.test", - ); - }); - - it("silently falls back without retrying a failed warmup", async () => { - const prepareConnection = vi.fn().mockRejectedValue(new Error("offline")); - const prewarmer = new SfuStandbyPrewarmer(async () => ({ - Room: class { - prepareConnection = prepareConnection; - }, - })); - - prewarmer.setUrl("wss://sfu.example.test"); - await vi.waitFor(() => expect(prepareConnection).toHaveBeenCalledOnce()); - prewarmer.setUrl("wss://sfu.example.test"); - await Promise.resolve(); - - expect(prepareConnection).toHaveBeenCalledOnce(); - }); - - it("does not reschedule a failed SDK import for the same URL", async () => { - const load = vi.fn().mockRejectedValue(new Error("chunk unavailable")); - const prewarmer = new SfuStandbyPrewarmer(load); - - prewarmer.setUrl("wss://sfu.example.test"); - await vi.waitFor(() => expect(load).toHaveBeenCalledOnce()); - await Promise.resolve(); - prewarmer.setUrl("wss://sfu.example.test"); - - expect(load).toHaveBeenCalledOnce(); - }); -}); diff --git a/tsconfig.tools.json b/tsconfig.tools.json index d98680c..6b8f76d 100644 --- a/tsconfig.tools.json +++ b/tsconfig.tools.json @@ -23,8 +23,8 @@ "scripts/client-browser-fanout-gate.ts", "scripts/client-media-gate.ts", "scripts/client-native-host-gate.ts", - "scripts/client-native-sfu-gate.ts", - "scripts/client-native-sfu-page.ts", + "scripts/embedded-sfu-gate.ts", + "scripts/embedded-sfu-page.ts", "scripts/client-loopback-gate.ts", "scripts/client-local-gate.ts", "scripts/client-link-gate.ts",