AAA
1332473513
Rename the active product, Go module, binaries, configuration, protocols and documentation to Piik while retaining the TV mascot and existing media/room behavior. Use one current private contract without old-name aliases. Preserve SQLite authority and copy compatible local settings once; other settings start fresh. Git-message cleanup and the infrastructure name cutover are coordinated with this phase, with original history and deployment data archived externally. Validated locally with Web and Go checks, Windows capture compilation, matched Client/Server builds, packaged Client control handshake and history-rewrite integrity checks. Skip duplicate hosted CI for this owner-authorized local release. [skip ci]
652 lines
29 KiB
C++
652 lines
29 KiB
C++
#include "adaptive_encoder.h"
|
|
#include "capture_output.h"
|
|
|
|
#include <algorithm>
|
|
#include <cmath>
|
|
#include <exception>
|
|
#include <future>
|
|
#include <iostream>
|
|
#include <limits>
|
|
#include <mutex>
|
|
#include <syncstream>
|
|
#include <thread>
|
|
|
|
#include "api/environment/environment_factory.h"
|
|
#include "api/make_ref_counted.h"
|
|
#include "api/task_queue/task_queue_base.h"
|
|
#include "api/task_queue/task_queue_factory.h"
|
|
#include "api/video/builtin_video_bitrate_allocator_factory.h"
|
|
#include "api/video/i420_buffer.h"
|
|
#include "api/video/video_adapter.h"
|
|
#include "api/video/video_frame.h"
|
|
#include "api/video/video_stream_encoder_settings.h"
|
|
#include "api/video_codecs/builtin_video_encoder_factory.h"
|
|
#include "api/video_codecs/video_encoder_factory.h"
|
|
#include "call/video_send_stream.h"
|
|
#include "libyuv/convert.h"
|
|
#include "modules/video_coding/include/video_error_codes.h"
|
|
#include "system_wrappers/include/clock.h"
|
|
#include "video/adaptation/overuse_frame_detector.h"
|
|
#include "video/frame_cadence_adapter.h"
|
|
#include "video/send_statistics_proxy.h"
|
|
#include "video/video_stream_encoder.h"
|
|
|
|
namespace piik::capture::windows {
|
|
namespace {
|
|
|
|
template <typename F>
|
|
void Sync(webrtc::TaskQueueBase* queue, F fn) {
|
|
std::promise<void> completed;
|
|
auto result = completed.get_future();
|
|
queue->PostTask([completed = std::move(completed), fn = std::move(fn)]() mutable {
|
|
try {
|
|
fn();
|
|
completed.set_value();
|
|
} catch (...) {
|
|
completed.set_exception(std::current_exception());
|
|
}
|
|
});
|
|
result.get();
|
|
}
|
|
|
|
// Library callbacks cannot throw through WebRTC's task queues.
|
|
class Failure final {
|
|
public:
|
|
void Capture() noexcept {
|
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
if (!failure_) failure_ = std::current_exception();
|
|
}
|
|
void Set(const char* detail) noexcept {
|
|
try {
|
|
throw GateFailure("adaptive-encoder", detail);
|
|
} catch (...) {
|
|
Capture();
|
|
}
|
|
}
|
|
void Rethrow() {
|
|
std::exception_ptr failure;
|
|
{
|
|
std::lock_guard<std::mutex> lock(mutex_);
|
|
failure = failure_;
|
|
}
|
|
if (failure) {
|
|
try {
|
|
std::rethrow_exception(failure);
|
|
} catch (const GateFailure&) {
|
|
throw;
|
|
} catch (const std::exception& error) {
|
|
throw GateFailure("adaptive-encoder", error.what());
|
|
} catch (...) {
|
|
throw GateFailure("adaptive-encoder", "WebRTC encoder callback failed");
|
|
}
|
|
}
|
|
}
|
|
|
|
private:
|
|
std::mutex mutex_;
|
|
std::exception_ptr failure_;
|
|
};
|
|
|
|
class Readback final {
|
|
public:
|
|
Readback(ID3D11Device* device, Failure& failure) : device_(device), failure_(failure) {
|
|
device_->GetImmediateContext(&context_);
|
|
}
|
|
|
|
webrtc::scoped_refptr<webrtc::I420BufferInterface> Convert(ID3D11Texture2D* texture) {
|
|
try {
|
|
D3D11_TEXTURE2D_DESC description{};
|
|
texture->GetDesc(&description);
|
|
if (!staging_ || width_ != description.Width || height_ != description.Height) {
|
|
staging_.Reset();
|
|
description.Usage = D3D11_USAGE_STAGING;
|
|
description.BindFlags = 0;
|
|
description.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
|
|
description.MiscFlags = 0;
|
|
Check(device_->CreateTexture2D(&description, nullptr, &staging_),
|
|
"adaptive-vp8-staging");
|
|
width_ = description.Width;
|
|
height_ = description.Height;
|
|
}
|
|
auto pixels = webrtc::I420Buffer::Create(static_cast<int>(width_),
|
|
static_cast<int>(height_));
|
|
context_->CopyResource(staging_.Get(), texture);
|
|
D3D11_MAPPED_SUBRESOURCE mapped{};
|
|
Check(context_->Map(staging_.Get(), 0, D3D11_MAP_READ, 0, &mapped),
|
|
"adaptive-vp8-readback");
|
|
const auto* y = static_cast<const UINT8*>(mapped.pData);
|
|
const int result = libyuv::NV12ToI420(
|
|
y, static_cast<int>(mapped.RowPitch), y + mapped.RowPitch * height_,
|
|
static_cast<int>(mapped.RowPitch), pixels->MutableDataY(), pixels->StrideY(),
|
|
pixels->MutableDataU(), pixels->StrideU(), pixels->MutableDataV(),
|
|
pixels->StrideV(), static_cast<int>(width_), static_cast<int>(height_));
|
|
context_->Unmap(staging_.Get(), 0);
|
|
if (result != 0) Fail("adaptive-vp8-convert", "NV12 conversion failed");
|
|
return pixels;
|
|
} catch (...) {
|
|
failure_.Capture();
|
|
return nullptr;
|
|
}
|
|
}
|
|
|
|
private:
|
|
ComPtr<ID3D11Device> device_;
|
|
ComPtr<ID3D11DeviceContext> context_;
|
|
ComPtr<ID3D11Texture2D> staging_;
|
|
Failure& failure_;
|
|
UINT32 width_ = 0, height_ = 0;
|
|
};
|
|
|
|
class NativeVideoFrameBuffer : public webrtc::VideoFrameBuffer {
|
|
public:
|
|
NativeVideoFrameBuffer(ComPtr<ID3D11Texture2D> texture, UINT32 width, UINT32 height,
|
|
UINT64 timestamp100ns, Readback& readback)
|
|
: texture_(std::move(texture)), width_(width), height_(height),
|
|
timestamp100ns_(timestamp100ns), readback_(readback) {}
|
|
Type type() const override { return Type::kNative; }
|
|
int width() const override { return static_cast<int>(width_); }
|
|
int height() const override { return static_cast<int>(height_); }
|
|
webrtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
|
|
return readback_.Convert(texture_.Get());
|
|
}
|
|
ID3D11Texture2D* texture() const { return texture_.Get(); }
|
|
UINT64 timestamp100ns() const { return timestamp100ns_; }
|
|
|
|
private:
|
|
ComPtr<ID3D11Texture2D> texture_;
|
|
const UINT32 width_, height_;
|
|
const UINT64 timestamp100ns_;
|
|
Readback& readback_;
|
|
};
|
|
|
|
class HardwareEncoder final : public webrtc::VideoEncoder {
|
|
public:
|
|
HardwareEncoder(const webrtc::Environment& env, VideoProfile ceiling,
|
|
AdaptiveEncoder::Factory create,
|
|
std::unique_ptr<piik::capture::windows::VideoEncoder> initial,
|
|
Failure& failure)
|
|
: env_(env), ceiling_(ceiling), create_(std::move(create)),
|
|
initial_(std::move(initial)), failure_(failure) {
|
|
Check(CoInitializeEx(nullptr, COINIT_MULTITHREADED), "adaptive-h264-apartment");
|
|
}
|
|
~HardwareEncoder() override {
|
|
initial_.reset();
|
|
Release();
|
|
CoUninitialize();
|
|
}
|
|
|
|
int32_t InitEncode(const webrtc::VideoCodec* config, const Settings&) override {
|
|
Release();
|
|
try {
|
|
if (!config || config->codecType != webrtc::kVideoCodecH264 ||
|
|
config->width == 0 || config->height == 0 || (config->width & 1) ||
|
|
(config->height & 1) || config->width > ceiling_.width ||
|
|
config->height > ceiling_.height || config->maxFramerate == 0 ||
|
|
config->maxFramerate > ceiling_.frame_rate || config->startBitrate == 0 ||
|
|
config->numberOfSimulcastStreams > 1 || config->H264().numberOfTemporalLayers > 1) {
|
|
Fail("adaptive-h264-config", "WebRTC requested an unsupported H264 configuration");
|
|
}
|
|
profile_ = ceiling_;
|
|
profile_.width = config->width;
|
|
profile_.height = config->height;
|
|
profile_.frame_rate = config->maxFramerate;
|
|
profile_.bit_rate = config->startBitrate * 1000;
|
|
if (initial_ && profile_.width == ceiling_.width &&
|
|
profile_.height == ceiling_.height && profile_.frame_rate == ceiling_.frame_rate) {
|
|
encoder_ = std::move(initial_);
|
|
encoder_->SetBitrate(profile_.bit_rate);
|
|
} else {
|
|
initial_.reset();
|
|
encoder_ = create_(profile_);
|
|
}
|
|
if (!encoder_ || encoder_->kind != OutputKind::h264)
|
|
Fail("adaptive-h264-create", "The native H264 encoder is unavailable");
|
|
applied_bitrate_ = profile_.bit_rate;
|
|
return WEBRTC_VIDEO_CODEC_OK;
|
|
} catch (...) {
|
|
failure_.Capture();
|
|
Release();
|
|
return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
|
|
}
|
|
}
|
|
|
|
int32_t RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback* callback) override {
|
|
callback_ = callback;
|
|
return WEBRTC_VIDEO_CODEC_OK;
|
|
}
|
|
int32_t Release() override {
|
|
encoder_.reset();
|
|
paused_ = false;
|
|
key_frame_pending_ = true;
|
|
applied_bitrate_ = 0;
|
|
return WEBRTC_VIDEO_CODEC_OK;
|
|
}
|
|
|
|
void SetRates(const RateControlParameters& rates) override {
|
|
if (!encoder_) return;
|
|
try {
|
|
const UINT32 bitrate = rates.bitrate.get_sum_bps();
|
|
if (bitrate != rates.bitrate.GetBitrate(0, 0) ||
|
|
!std::isfinite(rates.framerate_fps))
|
|
Fail("adaptive-h264-rates", "WebRTC supplied unsupported H264 rates");
|
|
if (bitrate == 0) {
|
|
paused_ = true;
|
|
return;
|
|
}
|
|
const double fps = rates.framerate_fps > 0 ? rates.framerate_fps : profile_.frame_rate;
|
|
// Chromium M152 MF compensates bitrate when requested FPS differs from configured FPS.
|
|
const double adjusted = static_cast<double>(bitrate) * profile_.frame_rate / fps;
|
|
if (!std::isfinite(adjusted) || adjusted < 1 ||
|
|
adjusted > std::numeric_limits<UINT32>::max())
|
|
Fail("adaptive-h264-rates", "H264 rate compensation overflowed");
|
|
const auto next = static_cast<UINT32>(adjusted);
|
|
if (next != applied_bitrate_) encoder_->SetBitrate(next);
|
|
applied_bitrate_ = next;
|
|
key_frame_pending_ = key_frame_pending_ || paused_;
|
|
paused_ = false;
|
|
} catch (...) {
|
|
failure_.Capture();
|
|
}
|
|
}
|
|
|
|
int32_t Encode(const webrtc::VideoFrame& frame,
|
|
const std::vector<webrtc::VideoFrameType>* requested) override {
|
|
try {
|
|
failure_.Rethrow();
|
|
if (!encoder_ || !callback_)
|
|
Fail("adaptive-h264-encode", "The H264 encoder is not initialized");
|
|
key_frame_pending_ = key_frame_pending_ ||
|
|
(requested && std::find(requested->begin(), requested->end(),
|
|
webrtc::VideoFrameType::kVideoFrameKey) != requested->end());
|
|
if (paused_ || (requested && !requested->empty() &&
|
|
requested->front() == webrtc::VideoFrameType::kEmptyFrame)) {
|
|
callback_->OnFrameDropped(frame.rtp_timestamp(), 0, true);
|
|
return WEBRTC_VIDEO_CODEC_OK;
|
|
}
|
|
const auto* native = dynamic_cast<NativeVideoFrameBuffer*>(frame.video_frame_buffer().get());
|
|
if (!native || frame.width() != static_cast<int>(profile_.width) ||
|
|
frame.height() != static_cast<int>(profile_.height))
|
|
Fail("adaptive-h264-input", "H264 requires the selected native NV12 surface");
|
|
const auto began = env_.clock().TimeInMilliseconds();
|
|
const auto access_unit = encoder_->Encode(native->texture(), native->timestamp100ns(),
|
|
key_frame_pending_);
|
|
key_frame_pending_ = false;
|
|
webrtc::EncodedImage image;
|
|
image.SetEncodedData(webrtc::EncodedImageBuffer::Create(
|
|
access_unit.bytes.data(), access_unit.bytes.size()));
|
|
image._encodedWidth = profile_.width;
|
|
image._encodedHeight = profile_.height;
|
|
image.SetRtpTimestamp(frame.rtp_timestamp());
|
|
image.capture_time_ms_ = frame.timestamp_us() / 1000;
|
|
image.ntp_time_ms_ = frame.ntp_time_ms();
|
|
image.SetPresentationTimestamp(frame.presentation_timestamp());
|
|
image.rotation_ = frame.rotation();
|
|
image.SetColorSpace(frame.color_space());
|
|
image.SetVideoFrameTrackingId(frame.id());
|
|
image.SetSimulcastIndex(0);
|
|
image.SetSpatialIndex(0);
|
|
image.SetTemporalIndex(0);
|
|
image.set_end_of_temporal_unit(true);
|
|
image.SetFrameType(access_unit.key_frame ? webrtc::VideoFrameType::kVideoFrameKey
|
|
: webrtc::VideoFrameType::kVideoFrameDelta);
|
|
image.SetEncodeTime(began, env_.clock().TimeInMilliseconds());
|
|
webrtc::CodecSpecificInfo info;
|
|
info.codecType = webrtc::kVideoCodecH264;
|
|
info.codecSpecific.H264 = {};
|
|
info.codecSpecific.H264.packetization_mode = webrtc::H264PacketizationMode::NonInterleaved;
|
|
info.codecSpecific.H264.idr_frame = access_unit.key_frame;
|
|
info.scalability_mode = webrtc::ScalabilityMode::kL1T1;
|
|
if (callback_->OnEncodedImage(image, &info).error != webrtc::EncodedImageCallback::Result::OK)
|
|
Fail("adaptive-h264-callback", "WebRTC rejected the encoded H264 frame");
|
|
return WEBRTC_VIDEO_CODEC_OK;
|
|
} catch (...) {
|
|
failure_.Capture();
|
|
return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
|
|
}
|
|
}
|
|
|
|
EncoderInfo GetEncoderInfo() const override {
|
|
EncoderInfo info;
|
|
info.implementation_name = "Windows Media Foundation H264";
|
|
info.is_hardware_accelerated = true;
|
|
info.enable_cpu_overuse_detection = true;
|
|
info.has_trusted_rate_controller = false;
|
|
// VSE parses H264 slice QP; use upstream H264 quality-scaler thresholds.
|
|
info.is_qp_trusted = true;
|
|
info.scaling_settings = ScalingSettings(24, 37);
|
|
info.requested_resolution_alignment = 2;
|
|
info.supports_native_handle = true;
|
|
info.supports_simulcast = false;
|
|
info.preferred_pixel_formats = {webrtc::VideoFrameBuffer::Type::kNative};
|
|
return info;
|
|
}
|
|
|
|
private:
|
|
const webrtc::Environment env_;
|
|
const VideoProfile ceiling_;
|
|
AdaptiveEncoder::Factory create_;
|
|
std::unique_ptr<piik::capture::windows::VideoEncoder> initial_, encoder_;
|
|
Failure& failure_;
|
|
VideoProfile profile_;
|
|
webrtc::EncodedImageCallback* callback_ = nullptr;
|
|
UINT32 applied_bitrate_ = 0;
|
|
bool paused_ = false, key_frame_pending_ = true;
|
|
};
|
|
|
|
webrtc::DegradationPreference Preference(DegradationPreference preference) {
|
|
switch (preference) {
|
|
case DegradationPreference::resolution: return webrtc::DegradationPreference::MAINTAIN_RESOLUTION;
|
|
case DegradationPreference::framerate: return webrtc::DegradationPreference::MAINTAIN_FRAMERATE;
|
|
case DegradationPreference::balanced: return webrtc::DegradationPreference::BALANCED;
|
|
}
|
|
Fail("adaptive-preference", "Unknown degradation preference");
|
|
}
|
|
|
|
} // namespace
|
|
|
|
class AdaptiveEncoder::Impl final : public webrtc::VideoSourceInterface<webrtc::VideoFrame>,
|
|
public webrtc::VideoStreamEncoderInterface::EncoderSink,
|
|
public webrtc::VideoEncoderFactory {
|
|
public:
|
|
Impl(OutputKind kind, VideoProfile ceiling, ID3D11Device* device,
|
|
Factory create, std::unique_ptr<VideoEncoder> initial, int output_index)
|
|
: kind_(kind), ceiling_(ceiling), output_index_(output_index), device_(device), create_(std::move(create)),
|
|
initial_(std::move(initial)), env_(webrtc::CreateEnvironment()),
|
|
readback_(device, failure_), builtin_(webrtc::CreateBuiltinVideoEncoderFactory()),
|
|
allocator_(webrtc::CreateBuiltinVideoBitrateAllocatorFactory()),
|
|
worker_(env_.task_queue_factory().CreateTaskQueue(
|
|
"CaptureAdaptation", webrtc::TaskQueueFactory::Priority::kNormal)) {}
|
|
|
|
~Impl() override {
|
|
Sync(worker_.get(), [&] {
|
|
if (encoder_) {
|
|
encoder_->Stop();
|
|
encoder_.reset();
|
|
encode_queue_ = nullptr;
|
|
}
|
|
});
|
|
}
|
|
|
|
void Start() {
|
|
Sync(worker_.get(), [&] {
|
|
const auto format = GetSupportedFormats().front();
|
|
webrtc::VideoSendStream::Config send_config(nullptr);
|
|
send_config.rtp.ssrcs = {1};
|
|
send_config.rtp.payload_name = format.name;
|
|
send_config.rtp.payload_type = 96;
|
|
stats_ = std::make_unique<webrtc::SendStatisticsProxy>(
|
|
&env_.clock(), send_config, webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
|
|
env_.field_trials());
|
|
webrtc::VideoStreamEncoderSettings settings(webrtc::VideoEncoder::Capabilities(false));
|
|
settings.encoder_factory = this;
|
|
settings.bitrate_allocator_factory = allocator_.get();
|
|
auto queue = env_.task_queue_factory().CreateTaskQueue(
|
|
"CaptureEncoder", webrtc::TaskQueueFactory::Priority::kNormal);
|
|
encode_queue_ = queue.get();
|
|
auto cadence = webrtc::FrameCadenceAdapterInterface::Create(
|
|
&env_.clock(), queue.get(), nullptr, webrtc::TaskQueueBase::Current(), env_.field_trials());
|
|
encoder_ = std::make_unique<webrtc::VideoStreamEncoder>(
|
|
env_, static_cast<int>(std::max(1u, std::thread::hardware_concurrency())),
|
|
stats_.get(), settings, std::make_unique<webrtc::OveruseFrameDetector>(env_, stats_.get()),
|
|
std::move(cadence), std::move(queue),
|
|
webrtc::VideoStreamEncoder::BitrateAllocationCallbackType::kVideoBitrateAllocation,
|
|
nullptr, [this](std::optional<webrtc::SdpVideoFormat>, bool) {
|
|
failure_.Set("WebRTC could not continue with the selected encoder");
|
|
});
|
|
adapter_.OnOutputFormatRequest(std::nullopt,
|
|
static_cast<int>(ceiling_.width * ceiling_.height), static_cast<int>(ceiling_.frame_rate));
|
|
encoder_->SetSink(this, false);
|
|
encoder_->SetSource(this, Preference(ceiling_.preference));
|
|
encoder_->SetStartBitrate(static_cast<int>(ceiling_.bit_rate));
|
|
webrtc::VideoEncoderConfig config;
|
|
config.codec_type = kind_ == OutputKind::h264 ? webrtc::kVideoCodecH264 : webrtc::kVideoCodecVP8;
|
|
config.video_format = format;
|
|
config.content_type = webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo;
|
|
config.number_of_streams = 1;
|
|
config.max_bitrate_bps = static_cast<int>(ceiling_.bit_rate);
|
|
config.frame_drop_enabled = true;
|
|
config.is_quality_scaling_allowed = true;
|
|
config.simulcast_layers.resize(1);
|
|
config.simulcast_layers[0].active = true;
|
|
config.simulcast_layers[0].max_framerate = static_cast<int>(ceiling_.frame_rate);
|
|
config.simulcast_layers[0].max_bitrate_bps = static_cast<int>(ceiling_.bit_rate);
|
|
config.simulcast_layers[0].num_temporal_layers = 1;
|
|
if (kind_ == OutputKind::vp8) {
|
|
auto vp8 = webrtc::VideoEncoder::GetDefaultVp8Settings();
|
|
vp8.automaticResizeOn = true;
|
|
config.encoder_specific_settings =
|
|
webrtc::make_ref_counted<webrtc::VideoEncoderConfig::Vp8EncoderSpecificSettings>(vp8);
|
|
}
|
|
encoder_->ConfigureEncoder(std::move(config), 1200, [this](webrtc::RTCError error) {
|
|
if (!error.ok()) failure_.Set("WebRTC encoder configuration failed");
|
|
});
|
|
});
|
|
Sync(encode_queue_, [] {});
|
|
failure_.Rethrow();
|
|
}
|
|
|
|
std::optional<AdaptiveAccessUnit> Encode(FrameProducer produce, UINT32 source_width,
|
|
UINT32 source_height, UINT64 timestamp100ns, bool key_frame, UINT32 bitrate) {
|
|
if (!produce || source_width == 0 || source_height == 0 ||
|
|
source_width > ceiling_.width || source_height > ceiling_.height ||
|
|
timestamp100ns > static_cast<UINT64>(std::numeric_limits<INT64>::max()))
|
|
Fail("adaptive-input", "Invalid adaptive encoder input");
|
|
failure_.Rethrow();
|
|
Sync(worker_.get(), [&] {
|
|
const auto rate = webrtc::DataRate::BitsPerSec(std::min(bitrate, ceiling_.bit_rate));
|
|
if (!bitrate_ || *bitrate_ != rate.bps()) {
|
|
encoder_->OnBitrateUpdated(rate, rate, 0, 0, 0);
|
|
bitrate_ = rate.bps();
|
|
}
|
|
if (key_frame) encoder_->SendKeyFrame();
|
|
if (!source_sink_) return;
|
|
if (!anchor100ns_) {
|
|
anchor100ns_ = timestamp100ns;
|
|
anchor_us_ = env_.clock().TimeInMicroseconds();
|
|
anchor_ntp_ms_ = env_.clock().CurrentNtpInMilliseconds();
|
|
}
|
|
if (timestamp100ns < *anchor100ns_)
|
|
Fail("adaptive-timestamp", "Capture timestamp moved before its output anchor");
|
|
const auto elapsed = timestamp100ns - *anchor100ns_;
|
|
const INT64 capture_us = anchor_us_ + static_cast<INT64>(elapsed / 10);
|
|
const INT64 ntp_ms = anchor_ntp_ms_ + static_cast<INT64>(elapsed / 10'000);
|
|
int crop_width, crop_height, width, height;
|
|
if (!adapter_.AdaptFrameResolution(static_cast<int>(source_width),
|
|
static_cast<int>(source_height), capture_us * 1000,
|
|
&crop_width, &crop_height, &width, &height)) {
|
|
source_sink_->OnDiscardedFrame();
|
|
return;
|
|
}
|
|
auto texture = produce(static_cast<UINT32>(width), static_cast<UINT32>(height));
|
|
if (!texture) Fail("adaptive-surface", "The selected NV12 surface is unavailable");
|
|
D3D11_TEXTURE2D_DESC description{};
|
|
texture->GetDesc(&description);
|
|
ComPtr<ID3D11Device> texture_device;
|
|
texture->GetDevice(&texture_device);
|
|
if (texture_device.Get() != device_.Get() || description.Format != DXGI_FORMAT_NV12 ||
|
|
description.Width != static_cast<UINT32>(width) ||
|
|
description.Height != static_cast<UINT32>(height))
|
|
Fail("adaptive-surface", "The selected NV12 surface has a different device or size");
|
|
{
|
|
std::lock_guard<std::mutex> lock(output_mutex_);
|
|
timestamp100ns_ = timestamp100ns;
|
|
rtp_timestamp_ = 90u * static_cast<UINT32>(ntp_ms);
|
|
const double fps = std::min<double>(ceiling_.frame_rate, adapter_.GetMaxFramerate());
|
|
duration100ns_ = static_cast<UINT64>(10'000'000 / fps);
|
|
}
|
|
auto buffer = webrtc::make_ref_counted<NativeVideoFrameBuffer>(
|
|
std::move(texture), static_cast<UINT32>(width), static_cast<UINT32>(height),
|
|
timestamp100ns, readback_);
|
|
source_sink_->OnFrame(webrtc::VideoFrame::Builder().set_video_frame_buffer(buffer)
|
|
.set_timestamp_us(capture_us).set_ntp_time_ms(ntp_ms).build());
|
|
});
|
|
// Both selected codecs complete synchronously; native frames cannot be retained by VSE.
|
|
Sync(encode_queue_, [] {});
|
|
Sync(worker_.get(), [&] { TraceStats(); });
|
|
failure_.Rethrow();
|
|
std::lock_guard<std::mutex> lock(output_mutex_);
|
|
return std::exchange(output_, std::nullopt);
|
|
}
|
|
|
|
void AddOrUpdateSink(webrtc::VideoSinkInterface<webrtc::VideoFrame>* sink,
|
|
const webrtc::VideoSinkWants& wants) override {
|
|
source_sink_ = sink;
|
|
adapter_.OnSinkWants(wants);
|
|
}
|
|
void RemoveSink(webrtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override {
|
|
if (source_sink_ == sink) source_sink_ = nullptr;
|
|
}
|
|
|
|
std::vector<webrtc::SdpVideoFormat> GetSupportedFormats() const override {
|
|
if (kind_ == OutputKind::vp8) return {webrtc::SdpVideoFormat("VP8")};
|
|
return {webrtc::SdpVideoFormat("H264", {{"profile-level-id", ceiling_.profile_level_id()},
|
|
{"packetization-mode", "1"}, {"level-asymmetry-allowed", "1"}})};
|
|
}
|
|
std::unique_ptr<webrtc::VideoEncoder> Create(const webrtc::Environment& env,
|
|
const webrtc::SdpVideoFormat& format) override {
|
|
try {
|
|
if (kind_ == OutputKind::vp8) {
|
|
initial_.reset();
|
|
auto encoder = builtin_->Create(env, format);
|
|
if (!encoder) Fail("adaptive-vp8-create", "WebRTC VP8 encoder is unavailable");
|
|
return encoder;
|
|
}
|
|
return std::make_unique<HardwareEncoder>(env, ceiling_, create_, std::move(initial_), failure_);
|
|
} catch (...) {
|
|
failure_.Capture();
|
|
return nullptr;
|
|
}
|
|
}
|
|
|
|
void OnEncoderConfigurationChanged(std::vector<webrtc::VideoStream>, bool,
|
|
webrtc::VideoEncoderConfig::ContentType, int) override {}
|
|
void OnBitrateAllocationUpdated(const webrtc::VideoBitrateAllocation&) override {}
|
|
void OnVideoLayersAllocationUpdated(webrtc::VideoLayersAllocation) override {}
|
|
void OnFrameDropped(uint32_t, int, bool) override {}
|
|
Result OnEncodedImage(const webrtc::EncodedImage& image,
|
|
const webrtc::CodecSpecificInfo*) override {
|
|
try {
|
|
std::lock_guard<std::mutex> lock(output_mutex_);
|
|
if (output_)
|
|
Fail("adaptive-output-duplicate", "WebRTC returned more than one access unit for the current input");
|
|
if (image.RtpTimestamp() != rtp_timestamp_)
|
|
Fail("adaptive-output-timestamp", "WebRTC returned an access unit from a different input");
|
|
if (image.size() == 0 || image.size() > kMaxProductAccessUnitBytes)
|
|
Fail("adaptive-output-size", "Encoded access unit bytes=" + std::to_string(image.size()));
|
|
if (image._encodedWidth == 0 || image._encodedHeight == 0 ||
|
|
image._encodedWidth > ceiling_.width || image._encodedHeight > ceiling_.height)
|
|
Fail("adaptive-output-dimensions", "WebRTC returned output outside its configured dimensions");
|
|
AdaptiveAccessUnit output;
|
|
output.access_unit.timestamp100ns = timestamp100ns_;
|
|
output.access_unit.key_frame = image.IsKey();
|
|
output.access_unit.bytes.assign(image.data(), image.data() + image.size());
|
|
output.width = image._encodedWidth;
|
|
output.height = image._encodedHeight;
|
|
output.duration100ns = duration100ns_;
|
|
output_ = std::move(output);
|
|
return Result(Result::OK);
|
|
} catch (...) {
|
|
failure_.Capture();
|
|
return Result(Result::ERROR_SEND_FAILED);
|
|
}
|
|
}
|
|
|
|
private:
|
|
void TraceStats() {
|
|
// Read the existing VSE observer on Encode's cadence; diagnostics never drive adaptation.
|
|
if (!debug_) return;
|
|
const auto now = env_.clock().TimeInMilliseconds();
|
|
if (last_stats_ms_ && now - *last_stats_ms_ < 2'000) return;
|
|
last_stats_ms_ = now;
|
|
const auto stats = stats_->GetStats();
|
|
std::osyncstream line(std::cerr);
|
|
line << "event=encoder-stats output=" << output_index_
|
|
<< " codec=" << (kind_ == OutputKind::h264 ? "h264" : "vp8")
|
|
<< " ceilingWidth=" << ceiling_.width << " ceilingHeight=" << ceiling_.height
|
|
<< " ceilingFps=" << ceiling_.frame_rate << " ceilingBitrate=" << ceiling_.bit_rate
|
|
<< " inputFps=" << stats.input_frame_rate << " encodeFps=" << stats.encode_frame_rate
|
|
<< " inputFrames=" << stats.frames << " encodedFrames=" << stats.frames_encoded
|
|
<< " encodeTimeMs=" << stats.total_encode_time_ms
|
|
<< " averageEncodeTimeMs=" << stats.avg_encode_time_ms
|
|
<< " encodeUsagePercent=" << stats.encode_usage_percent
|
|
<< " targetBitrate=" << stats.target_media_bitrate_bps
|
|
<< " encodedBitrate=" << stats.media_bitrate_bps
|
|
<< " droppedCapturer=" << stats.frames_dropped_by_capturer
|
|
<< " droppedTimestamp=" << stats.frames_dropped_by_bad_timestamp
|
|
<< " droppedQueue=" << stats.frames_dropped_by_encoder_queue
|
|
<< " droppedRateLimiter=" << stats.frames_dropped_by_rate_limiter
|
|
<< " droppedCongestionWindow=" << stats.frames_dropped_by_congestion_window
|
|
<< " droppedEncoder=" << stats.frames_dropped_by_encoder
|
|
<< " cpuResolutionLimited=" << stats.cpu_limited_resolution
|
|
<< " cpuFpsLimited=" << stats.cpu_limited_framerate
|
|
<< " bandwidthResolutionLimited=" << stats.bw_limited_resolution
|
|
<< " bandwidthFpsLimited=" << stats.bw_limited_framerate
|
|
<< " cpuAdaptations=" << stats.number_of_cpu_adapt_changes
|
|
<< " qualityAdaptations=" << stats.number_of_quality_adapt_changes
|
|
<< " resolutionChanges=" << stats.quality_limitation_resolution_changes
|
|
<< " suspended=" << stats.suspended;
|
|
for (const auto& [ssrc, stream] : stats.substreams) {
|
|
// This adapter has one local observer stream; no network identity is exported.
|
|
(void)ssrc;
|
|
line << " outputWidth=" << stream.width << " outputHeight=" << stream.height
|
|
<< " outputFrames=" << stream.frames_encoded;
|
|
if (stream.qp_sum) line << " qpSum=" << *stream.qp_sum;
|
|
}
|
|
line << '\n';
|
|
}
|
|
|
|
const OutputKind kind_;
|
|
const VideoProfile ceiling_;
|
|
const int output_index_;
|
|
const bool debug_ = [] {
|
|
wchar_t value[2]{};
|
|
return GetEnvironmentVariableW(L"PIIK_CAPTURE_DEBUG", value, 2) == 1 && value[0] == L'1';
|
|
}();
|
|
std::optional<INT64> last_stats_ms_;
|
|
ComPtr<ID3D11Device> device_;
|
|
Factory create_;
|
|
std::unique_ptr<VideoEncoder> initial_;
|
|
const webrtc::Environment env_;
|
|
Failure failure_;
|
|
Readback readback_;
|
|
std::unique_ptr<webrtc::VideoEncoderFactory> builtin_;
|
|
std::unique_ptr<webrtc::VideoBitrateAllocatorFactory> allocator_;
|
|
std::unique_ptr<webrtc::SendStatisticsProxy> stats_;
|
|
std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter> worker_;
|
|
std::unique_ptr<webrtc::VideoStreamEncoder> encoder_;
|
|
webrtc::TaskQueueBase* encode_queue_ = nullptr;
|
|
webrtc::VideoSinkInterface<webrtc::VideoFrame>* source_sink_ = nullptr;
|
|
webrtc::VideoAdapter adapter_{2};
|
|
std::optional<INT64> bitrate_;
|
|
std::optional<UINT64> anchor100ns_;
|
|
INT64 anchor_us_ = 0, anchor_ntp_ms_ = 0;
|
|
std::mutex output_mutex_;
|
|
std::optional<AdaptiveAccessUnit> output_;
|
|
UINT64 timestamp100ns_ = 0, duration100ns_ = 0;
|
|
UINT32 rtp_timestamp_ = 0;
|
|
};
|
|
|
|
AdaptiveEncoder::AdaptiveEncoder(OutputKind kind, VideoProfile ceiling, ID3D11Device* device,
|
|
Factory create, std::unique_ptr<VideoEncoder> initial, int output_index) {
|
|
if ((kind != OutputKind::vp8 && kind != OutputKind::h264) || !device ||
|
|
(kind == OutputKind::h264 && !create) ||
|
|
ceiling.width == 0 || ceiling.height == 0 || (ceiling.width & 1) || (ceiling.height & 1) ||
|
|
static_cast<UINT64>(ceiling.width) * ceiling.height > std::numeric_limits<int>::max() ||
|
|
ceiling.frame_rate == 0 || ceiling.frame_rate > 1'000 || ceiling.bit_rate < 1'000 ||
|
|
ceiling.bit_rate > static_cast<UINT32>(std::numeric_limits<int>::max()))
|
|
Fail("adaptive-profile", "Invalid adaptive encoder profile");
|
|
impl_ = std::make_unique<Impl>(kind, ceiling, device, std::move(create), std::move(initial), output_index);
|
|
impl_->Start();
|
|
}
|
|
|
|
AdaptiveEncoder::~AdaptiveEncoder() = default;
|
|
|
|
std::optional<AdaptiveAccessUnit> AdaptiveEncoder::Encode(FrameProducer produce,
|
|
UINT32 source_width, UINT32 source_height, UINT64 timestamp100ns,
|
|
bool key_frame, UINT32 bitrate) {
|
|
return impl_->Encode(std::move(produce), source_width, source_height, timestamp100ns,
|
|
key_frame, bitrate);
|
|
}
|
|
|
|
} // namespace piik::capture::windows
|