#include "h264_encoder_adapter.h" #include "../../capture/windows/h264_encoder.h" #include #include #include #include #include #include #include "api/video_codecs/h264_profile_level_id.h" #include "libyuv/convert_from.h" #include "modules/video_coding/include/video_codec_interface.h" #include "modules/video_coding/include/video_error_codes.h" #include "system_wrappers/include/clock.h" namespace piik::probe { namespace { namespace native = piik::capture::windows; bool ValidDimensions(int width, int height) { return width >= 2 && width <= 2560 && height >= 2 && height <= 1440 && width % 2 == 0 && height % 2 == 0; } webrtc::SdpVideoFormat HardwareFormat() { return {"H264", {{"profile-level-id", "42c033"}, {"level-asymmetry-allowed", "1"}, {"packetization-mode", "1"}}, {webrtc::ScalabilityMode::kL1T1}}; } class HardwareEncoder final : public webrtc::VideoEncoder { public: HardwareEncoder(const webrtc::Environment& env, unsigned adapter_index, unsigned encoder_index, webrtc::H264Level maximum_level) : env_(env), adapter_index_(adapter_index), encoder_index_(encoder_index), maximum_level_(maximum_level) {} ~HardwareEncoder() override { Release(); } int InitEncode(const webrtc::VideoCodec* config, const Settings&) override { Release(); if (!config || config->codecType != webrtc::kVideoCodecH264 || !ValidDimensions(config->width, config->height) || config->maxFramerate == 0 || config->maxFramerate > 60 || config->startBitrate == 0 || config->startBitrate > std::numeric_limits::max() / 1000) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } if (config->numberOfSimulcastStreams > 1 || config->H264().numberOfTemporalLayers > 1 || (config->GetScalabilityMode() && config->GetScalabilityMode() != webrtc::ScalabilityMode::kL1T1)) { return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED; } profile_.width = config->width; profile_.height = config->height; profile_.frame_rate = config->maxFramerate; profile_.bit_rate = config->startBitrate * 1000; if (profile_.h264_level() > static_cast(maximum_level_)) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } try { runtime_ = std::make_unique(); auto adapters = native::EnumerateAdapters(); const auto& adapter = native::SelectAdapter(adapters, adapter_index_); device_ = native::CreateDevice(adapter); auto activations = native::EnumerateHardwareEncoders(adapter); encoder_ = std::make_unique( native::ActivateTransform(activations, encoder_index_, device_.manager.Get()), profile_); nv12_.resize(static_cast(profile_.width) * profile_.height * 3 / 2); implementation_name_ = encoder_->name + " (" + encoder_->identity + ")"; applied_bitrate_ = profile_.bit_rate; content_type_ = config->mode == webrtc::VideoCodecMode::kScreensharing ? webrtc::VideoContentType::SCREENSHARE : webrtc::VideoContentType::UNSPECIFIED; return WEBRTC_VIDEO_CODEC_OK; } catch (const std::exception& error) { ReportFailure("initialize", error); Release(); return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; } } int32_t RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback* callback) override { callback_ = callback; return WEBRTC_VIDEO_CODEC_OK; } int32_t Release() override { encoder_.reset(); device_ = {}; runtime_.reset(); nv12_.clear(); applied_bitrate_ = 0; paused_ = false; key_frame_pending_ = true; error_ = WEBRTC_VIDEO_CODEC_OK; return WEBRTC_VIDEO_CODEC_OK; } void SetRates(const RateControlParameters& rates) override { if (!encoder_ || error_ != WEBRTC_VIDEO_CODEC_OK) return; const UINT32 bitrate = rates.bitrate.get_sum_bps(); if (bitrate != rates.bitrate.GetBitrate(0, 0) || !std::isfinite(rates.framerate_fps)) { error_ = WEBRTC_VIDEO_CODEC_ERR_PARAMETER; return; } if (bitrate == 0) { paused_ = true; return; } const double fps = rates.framerate_fps > 0 ? rates.framerate_fps : profile_.frame_rate; // Chromium M152 MF keeps the configured FPS and compensates its bitrate: // media/gpu/windows/media_foundation_video_encode_accelerator_win.cc, // AdjustBitrateToFrameRate (152.0.7977.82). const double adjusted = static_cast(bitrate) * profile_.frame_rate / fps; if (!std::isfinite(adjusted) || adjusted < 1 || adjusted > std::numeric_limits::max()) { error_ = WEBRTC_VIDEO_CODEC_ERR_PARAMETER; return; } const auto next_bitrate = static_cast(adjusted); try { if (next_bitrate != applied_bitrate_) encoder_->SetBitrate(next_bitrate); applied_bitrate_ = next_bitrate; key_frame_pending_ = key_frame_pending_ || paused_; paused_ = false; } catch (const std::exception& error) { ReportFailure("rates", error); error_ = WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; } } int32_t Encode(const webrtc::VideoFrame& frame, const std::vector* requested) override { if (!encoder_ || !callback_) return WEBRTC_VIDEO_CODEC_UNINITIALIZED; if (error_ != WEBRTC_VIDEO_CODEC_OK) return error_; key_frame_pending_ = key_frame_pending_ || (requested && std::find(requested->begin(), requested->end(), webrtc::VideoFrameType::kVideoFrameKey) != requested->end()); if (paused_ || (requested && !requested->empty() && requested->front() == webrtc::VideoFrameType::kEmptyFrame)) { callback_->OnFrameDropped(frame.rtp_timestamp(), 0, true); return WEBRTC_VIDEO_CODEC_OK; } if (!frame.video_frame_buffer() || frame.width() != static_cast(profile_.width) || frame.height() != static_cast(profile_.height) || frame.timestamp_us() < 0 || frame.timestamp_us() > std::numeric_limits::max() / 10) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } try { const auto began = env_.clock().TimeInMilliseconds(); auto input = frame.video_frame_buffer()->ToI420(); if (!input || libyuv::I420ToNV12( input->DataY(), input->StrideY(), input->DataU(), input->StrideU(), input->DataV(), input->StrideV(), nv12_.data(), profile_.width, nv12_.data() + static_cast(profile_.width) * profile_.height, profile_.width, profile_.width, profile_.height) != 0) { return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; } D3D11_TEXTURE2D_DESC description{}; description.Width = profile_.width; description.Height = profile_.height; description.MipLevels = 1; description.ArraySize = 1; description.Format = DXGI_FORMAT_NV12; description.SampleDesc.Count = 1; description.Usage = D3D11_USAGE_DEFAULT; D3D11_SUBRESOURCE_DATA pixels{}; pixels.pSysMem = nv12_.data(); pixels.SysMemPitch = profile_.width; pixels.SysMemSlicePitch = static_cast(nv12_.size()); native::ComPtr texture; native::Check(device_.device->CreateTexture2D(&description, &pixels, &texture), "webrtc-h264-upload"); const auto access_unit = encoder_->Encode( texture.Get(), static_cast(frame.timestamp_us()) * 10, key_frame_pending_); key_frame_pending_ = false; webrtc::EncodedImage image; image.SetEncodedData(webrtc::EncodedImageBuffer::Create( access_unit.bytes.data(), access_unit.bytes.size())); image._encodedWidth = profile_.width; image._encodedHeight = profile_.height; image.SetRtpTimestamp(frame.rtp_timestamp()); image.capture_time_ms_ = frame.timestamp_us() / 1000; image.ntp_time_ms_ = frame.ntp_time_ms(); image.SetPresentationTimestamp(frame.presentation_timestamp()); image.rotation_ = frame.rotation(); image.content_type_ = content_type_; image.SetColorSpace(frame.color_space()); image.SetPacketInfos(frame.packet_infos()); image.SetVideoFrameTrackingId(frame.id()); image.SetSimulcastIndex(0); image.SetSpatialIndex(0); image.SetTemporalIndex(0); image.set_end_of_temporal_unit(true); image.SetFrameType(access_unit.key_frame ? webrtc::VideoFrameType::kVideoFrameKey : webrtc::VideoFrameType::kVideoFrameDelta); image.SetEncodeTime(began, env_.clock().TimeInMilliseconds()); webrtc::CodecSpecificInfo info; info.codecType = webrtc::kVideoCodecH264; info.codecSpecific.H264 = {}; info.codecSpecific.H264.packetization_mode = webrtc::H264PacketizationMode::NonInterleaved; info.codecSpecific.H264.idr_frame = access_unit.key_frame; info.scalability_mode = webrtc::ScalabilityMode::kL1T1; const auto result = callback_->OnEncodedImage(image, &info); return result.error == webrtc::EncodedImageCallback::Result::OK ? WEBRTC_VIDEO_CODEC_OK : WEBRTC_VIDEO_CODEC_ERROR; } catch (const std::exception& error) { ReportFailure("encode", error); error_ = WEBRTC_VIDEO_CODEC_ENCODER_FAILURE; return error_; } } EncoderInfo GetEncoderInfo() const override { EncoderInfo info; info.implementation_name = implementation_name_; info.is_hardware_accelerated = true; info.enable_cpu_overuse_detection = true; info.has_trusted_rate_controller = false; info.is_qp_trusted = false; info.scaling_settings = ScalingSettings::kOff; info.requested_resolution_alignment = 2; info.supports_native_handle = false; info.supports_simulcast = false; info.preferred_pixel_formats = {webrtc::VideoFrameBuffer::Type::kI420}; return info; } private: static void ReportFailure(const char* operation, const std::exception& error) { std::cerr << "hardware-h264 " << operation << ": " << error.what(); if (const auto* failure = dynamic_cast(&error)) { std::cerr << " stage=" << failure->stage() << " hresult=" << failure->result(); } std::cerr << '\n'; } const webrtc::Environment env_; const unsigned adapter_index_; const unsigned encoder_index_; const webrtc::H264Level maximum_level_; std::unique_ptr runtime_; native::DeviceContext device_; std::unique_ptr encoder_; native::VideoProfile profile_; std::vector nv12_; webrtc::EncodedImageCallback* callback_ = nullptr; std::string implementation_name_ = "Windows Media Foundation H264"; webrtc::VideoContentType content_type_ = webrtc::VideoContentType::UNSPECIFIED; UINT32 applied_bitrate_ = 0; int32_t error_ = WEBRTC_VIDEO_CODEC_OK; bool paused_ = false; bool key_frame_pending_ = true; }; class HardwareFactory final : public webrtc::VideoEncoderFactory { public: HardwareFactory(unsigned adapter_index, unsigned encoder_index) : adapter_index_(adapter_index), encoder_index_(encoder_index) {} std::vector GetSupportedFormats() const override { return {HardwareFormat()}; } CodecSupport QueryCodecSupport(const webrtc::SdpVideoFormat& format, std::optional scalability_mode, std::optional resolution) const override { if (resolution && !ValidDimensions(resolution->width, resolution->height)) return {}; const auto profile = webrtc::ParseSdpForH264ProfileLevelId(format.parameters); native::VideoProfile minimum; minimum.frame_rate = 1; if (resolution) { minimum.width = resolution->width; minimum.height = resolution->height; } const bool supported = format.IsSameCodec(HardwareFormat()) && (!scalability_mode || *scalability_mode == "L1T1") && profile && minimum.h264_level() <= static_cast(profile->level); return {supported, supported}; } std::unique_ptr Create( const webrtc::Environment& env, const webrtc::SdpVideoFormat& format) override { if (!QueryCodecSupport(format, std::nullopt, std::nullopt).is_supported) return nullptr; const auto profile = webrtc::ParseSdpForH264ProfileLevelId(format.parameters); return std::make_unique(env, adapter_index_, encoder_index_, profile->level); } private: const unsigned adapter_index_; const unsigned encoder_index_; }; } // namespace std::unique_ptr CreateHardwareEncoderFactory( unsigned adapter_index, unsigned encoder_index) { return std::make_unique(adapter_index, encoder_index); } } // namespace piik::probe