okooooooo/Piik
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(media): integrate embedded media services and shared native encoding

Browse Source 
Embed STUN and SFU in the shared Go process, reuse native encoding pipelines with independent adaptation, and complete Client lifecycle, settings, diagnostics and packaging fixes. Retain standard Browser senders after measured encoded-reuse limitations.
This commit is contained in:
AAA authored and GitHub committed 2026-09-09 06:09:06 +08:00
1 parent 61a9b74cbd
commit cfc323adce
274 files changed
+37854 -13238
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

native/capture/darwin/main.swift
+741 -174
View File
File diff suppressed because it is too large. Load diff
native/capture/linux/main.c
+549 -142
View File
@@ -1,6 +1,8 @@
#include <gst/app/gstappsink.h>
#include <gst/app/gstappsrc.h>
#include <gst/gst.h>
#include <gst/video/video-event.h>
#include <glib-unix.h>
#include <errno.h>
#include <stdint.h>
@@ -13,7 +15,8 @@
#include "portal.h"
enum {
kCaptureProtocol = 4,
kCaptureProtocol = 7,
kMaxOutputs = 6,
kMaxPayloadBytes = 4 * 1024 * 1024,
kAudioFrameBytes = 960 * 2 * 2,
};
@@ -38,21 +41,50 @@ typedef struct {
guint bit_rate_divisor;
} EncoderInfo;
typedef struct CaptureRun CaptureRun;
typedef struct {
CaptureRun *run;
GstElement *encoder;
GstElement *branch;
GstElement *gate;
VideoProfile profile;
guint layer;
gboolean enabled;
gboolean decodable;
guint64 key_requested;
guint64 key_sent;
guint64 key_acknowledged;
GstClockTime key_timestamp;
GstClockTime activation_timestamp;
} VideoOutput;
struct CaptureRun {
GMainLoop *loop;
GstElement *pipeline;
GstElement *encoder;
GstElement *encoded_source;
VideoProfile profile;
VideoOutput outputs[kMaxOutputs];
guint output_count;
guint original_output;
const EncoderInfo *encoder_info;
const char *restore_token;
GMutex lock;
char *failure;
gint stopping;
gboolean active;
guint64 last_timestamp;
GstClockTime last_input_timestamp;
guint8 input_header[32];
guint input_header_length;
GByteArray *input_payload;
guint input_size;
guint64 encoded_timestamp;
gboolean encoded_started;
GByteArray *audio;
guint64 audio_timestamp;
} CaptureRun;
};
static GMutex output_lock;
static gboolean write_all(const void *data, size_t size) {
const uint8_t *next = data;
@@ -79,15 +111,29 @@ static void put_u64_be(uint8_t *output, guint64 value) {
}
}
static gboolean write_frame(guint8 kind, guint8 flags, guint64 timestamp,
guint64 duration, const guint8 *payload,
gsize size) {
if (payload == NULL || size == 0 || size > kMaxPayloadBytes) return FALSE;
uint8_t header[28] = {'S', 'M', 'E', 'D', 1, kind, flags, 0};
static guint64 read_be(const guint8 *input, guint count) {
guint64 value = 0;
for (guint index = 0; index < count; ++index) value = (value << 8) | input[index];
return value;
}
static gboolean write_frame(guint8 kind, guint8 flags, guint8 layer,
guint width, guint height, guint64 timestamp,
guint64 duration, const guint8 *payload, gsize size) {
if ((kind != 5 && (payload == NULL || size == 0)) ||
size > ((kind == 3 || kind == 6) ? 1024 * 1024 : kMaxPayloadBytes)) return FALSE;
uint8_t header[32] = {'S', 'M', 'E', 'D', 2, kind, flags, layer};
put_u64_be(header + 8, timestamp);
put_u64_be(header + 16, duration);
put_u32_be(header + 24, (guint32)size);
return write_all(header, sizeof(header)) && write_all(payload, size);
header[24] = (guint8)(width >> 8);
header[25] = (guint8)width;
header[26] = (guint8)(height >> 8);
header[27] = (guint8)height;
put_u32_be(header + 28, (guint32)size);
g_mutex_lock(&output_lock);
gboolean written = write_all(header, sizeof(header)) && write_all(payload, size);
g_mutex_unlock(&output_lock);
return written;
}
static char *json_string(const char *value) {
@@ -129,7 +175,20 @@ static char *json_string(const char *value) {
}
static gboolean write_status(const char *json) {
return write_frame(3, 0, 0, 0, (const guint8 *)json, strlen(json));
return write_frame(3, 0, 0, 0, 0, 0, 0, (const guint8 *)json, strlen(json));
}
static char *output_profiles_json(const CaptureRun *run) {
GString *json = g_string_new("[");
for (guint index = 0; index < run->output_count; ++index) {
const VideoProfile *profile = &run->outputs[index].profile;
g_string_append_printf(json,
"%s{\"width\":%u,\"height\":%u,\"fps\":%u,\"bitrate\":%u}",
index == 0 ? "" : ",", profile->width, profile->height,
profile->frame_rate, profile->bit_rate);
}
g_string_append_c(json, ']');
return g_string_free(json, FALSE);
}
static void encoder_info_free(gpointer value) {
@@ -226,7 +285,7 @@ static gboolean has_factory(const char *name) {
}
static gboolean video_stack_available(void) {
const char *required[] = {"pipewiresrc", "queue", "videoconvert",
const char *required[] = {"pipewiresrc", "queue", "tee", "videoconvert",
"videoscale", "videorate", "h264parse",
"appsink"};
for (guint index = 0; index < G_N_ELEMENTS(required); ++index) {
@@ -309,9 +368,60 @@ static gboolean parse_uint(const char *value, guint minimum, guint maximum,
return TRUE;
}
static gboolean parse_profile(int count, char **values, VideoProfile *profile,
static gboolean parse_outputs(int count, char **values, guint start, CaptureRun *run) {
if (count <= (int)start || (count - (int)start) % 5 != 0 ||
(count - (int)start) / 5 > kMaxOutputs) return FALSE;
run->output_count = (guint)(count - (int)start) / 5;
for (guint index = 0; index < run->output_count; ++index) {
guint offset = start + index * 5;
VideoOutput *output = &run->outputs[index];
output->run = run;
output->layer = index;
output->enabled = strcmp(values[1], "--capture-video") == 0 && index < 2;
output->key_requested = 1;
output->key_timestamp = GST_CLOCK_TIME_NONE;
output->activation_timestamp = GST_CLOCK_TIME_NONE;
output->profile.preference = run->profile.preference;
if (strcmp(values[offset], "--output") != 0 ||
!parse_uint(values[offset + 1], 2, 2560, &output->profile.width) ||
!parse_uint(values[offset + 2], 2, 1440, &output->profile.height) ||
!parse_uint(values[offset + 3], 1, 60, &output->profile.frame_rate) ||
!parse_uint(values[offset + 4], 1000, 12000000, &output->profile.bit_rate) ||
output->profile.width % 2 != 0 || output->profile.height % 2 != 0) return FALSE;
}
return TRUE;
}
static gboolean parse_preference(const char *value, VideoProfile *profile) {
if (strcmp(value, "maintain-resolution") == 0) profile->preference = kPreferResolution;
else if (strcmp(value, "balanced") == 0) profile->preference = kPreferBalanced;
else if (strcmp(value, "maintain-framerate") == 0) profile->preference = kPreferFramerate;
else return FALSE;
return TRUE;
}
static gboolean parse_profile(int count, char **values, CaptureRun *run,
guint *encoder_index) {
if (count != 23 || strcmp(values[1], "--capture-video") != 0 ||
VideoProfile *profile = &run->profile;
if (count >= 16 && strcmp(values[1], "--encoded-video") == 0) {
if (strcmp(values[2], "--codec") != 0 || strcmp(values[3], "h264") != 0 ||
strcmp(values[4], "--adapter-index") != 0 || strcmp(values[5], "0") != 0 ||
strcmp(values[6], "--mft-index") != 0 || !parse_uint(values[7], 0, 63, encoder_index) ||
strcmp(values[8], "--preference") != 0 || !parse_preference(values[9], profile) ||
strcmp(values[10], "--protocol-v7") != 0 || !parse_outputs(count, values, 11, run)) return FALSE;
for (guint index = 0; index < run->output_count; ++index) {
const VideoProfile *output = &run->outputs[index].profile;
profile->width = MAX(profile->width, output->width);
profile->height = MAX(profile->height, output->height);
profile->frame_rate = MAX(profile->frame_rate, output->frame_rate);
profile->bit_rate = MAX(profile->bit_rate, output->bit_rate);
}
run->original_output = 0;
return TRUE;
}
if (count < 28 || (count - 23) % 5 != 0 ||
(count - 23) / 5 > kMaxOutputs ||
strcmp(values[1], "--capture-video") != 0 ||
strcmp(values[2], "picker") != 0 || strcmp(values[3], "1") != 0 ||
strcmp(values[4], "0") != 0 || strcmp(values[5], "0") != 0 ||
strcmp(values[6], "--adapter-index") != 0 ||
@@ -323,7 +433,7 @@ static gboolean parse_profile(int count, char **values, VideoProfile *profile,
strcmp(values[18], "--preference") != 0 ||
strcmp(values[20], "--codec") != 0 ||
(strcmp(values[21], "auto") != 0 && strcmp(values[21], "h264") != 0) ||
strcmp(values[22], "--protocol-v4") != 0 ||
strcmp(values[22], "--protocol-v7") != 0 ||
!parse_uint(values[9], 0, 63, encoder_index) ||
!parse_uint(values[11], 1, 16384, &profile->width) ||
!parse_uint(values[13], 1, 16384, &profile->height) ||
@@ -337,14 +447,16 @@ static gboolean parse_profile(int count, char **values, VideoProfile *profile,
(profile->width == 1920 && profile->height == 1080) ||
(profile->width == 2560 && profile->height == 1440);
if (!resolution) return FALSE;
if (strcmp(values[19], "maintain-resolution") == 0) {
profile->preference = kPreferResolution;
} else if (strcmp(values[19], "balanced") == 0) {
profile->preference = kPreferBalanced;
} else if (strcmp(values[19], "maintain-framerate") == 0) {
profile->preference = kPreferFramerate;
} else {
return FALSE;
if (!parse_preference(values[19], profile) || !parse_outputs(count, values, 23, run)) return FALSE;
run->original_output = MIN(1, run->output_count - 1);
const VideoProfile *original = &run->outputs[run->original_output].profile;
if (original->width != profile->width || original->height != profile->height ||
original->frame_rate != profile->frame_rate ||
original->bit_rate != profile->bit_rate) return FALSE;
for (guint index = 0; index < run->output_count; ++index) {
const VideoProfile *output = &run->outputs[index].profile;
if (output->width > profile->width || output->height > profile->height ||
output->frame_rate > profile->frame_rate || output->bit_rate > profile->bit_rate) return FALSE;
}
return TRUE;
}
@@ -355,17 +467,26 @@ static gboolean set_numeric_property(GObject *object, const char *name,
g_object_class_find_property(G_OBJECT_GET_CLASS(object), name);
if (property == NULL || !(property->flags & G_PARAM_WRITABLE)) return FALSE;
GType type = G_PARAM_SPEC_VALUE_TYPE(property);
GValue setting = G_VALUE_INIT;
g_value_init(&setting, type);
if (type == G_TYPE_UINT) {
g_object_set(object, name, (guint)value, NULL);
g_value_set_uint(&setting, (guint)value);
} else if (type == G_TYPE_INT) {
g_object_set(object, name, (gint)value, NULL);
g_value_set_int(&setting, (gint)value);
} else if (type == G_TYPE_UINT64) {
g_object_set(object, name, value, NULL);
g_value_set_uint64(&setting, value);
} else if (type == G_TYPE_INT64) {
g_object_set(object, name, (gint64)value, NULL);
g_value_set_int64(&setting, (gint64)value);
} else {
g_value_unset(&setting);
return FALSE;
}
if (g_param_value_validate(property, &setting)) {
g_value_unset(&setting);
return FALSE;
}
g_object_set_property(object, name, &setting);
g_value_unset(&setting);
return TRUE;
}
@@ -393,24 +514,25 @@ static void set_optional_enum(GObject *object, const char *name,
g_type_class_unref(values);
}
static gboolean configure_encoder(CaptureRun *run) {
static gboolean configure_encoder(VideoOutput *output) {
CaptureRun *run = output->run;
guint64 bitrate =
run->profile.bit_rate / run->encoder_info->bit_rate_divisor;
if (!set_numeric_property(G_OBJECT(run->encoder), "bitrate", bitrate)) {
output->profile.bit_rate / run->encoder_info->bit_rate_divisor;
if (!set_numeric_property(G_OBJECT(output->encoder), "bitrate", bitrate)) {
return FALSE;
}
set_numeric_property(G_OBJECT(run->encoder), "key-int-max",
run->profile.frame_rate * 2);
set_numeric_property(G_OBJECT(run->encoder), "b-frames", 0);
set_numeric_property(G_OBJECT(run->encoder), "ref-frames", 1);
set_optional_boolean(G_OBJECT(run->encoder), "cabac", FALSE);
set_optional_boolean(G_OBJECT(run->encoder), "aud", TRUE);
set_optional_boolean(G_OBJECT(run->encoder), "zerolatency", TRUE);
set_optional_enum(G_OBJECT(run->encoder), "rate-control", "cbr");
guint usage = run->profile.preference == kPreferResolution
set_numeric_property(G_OBJECT(output->encoder), "key-int-max",
output->profile.frame_rate * 2);
set_numeric_property(G_OBJECT(output->encoder), "b-frames", 0);
set_numeric_property(G_OBJECT(output->encoder), "ref-frames", 1);
set_optional_boolean(G_OBJECT(output->encoder), "cabac", FALSE);
set_optional_boolean(G_OBJECT(output->encoder), "aud", TRUE);
set_optional_boolean(G_OBJECT(output->encoder), "zerolatency", TRUE);
set_optional_enum(G_OBJECT(output->encoder), "rate-control", "cbr");
guint usage = output->profile.preference == kPreferResolution
? 1
: run->profile.preference == kPreferFramerate ? 7 : 4;
set_numeric_property(G_OBJECT(run->encoder), "target-usage", usage);
set_numeric_property(G_OBJECT(output->encoder), "target-usage", usage);
return TRUE;
}
@@ -480,40 +602,232 @@ static gboolean quit_loop(gpointer data) {
static void fail_run(CaptureRun *run, const char *message) {
g_mutex_lock(&run->lock);
if (run->failure == NULL) run->failure = g_strdup(message);
gboolean first = run->failure == NULL;
if (first) run->failure = g_strdup(message);
g_mutex_unlock(&run->lock);
g_main_context_invoke(NULL, quit_loop, run);
if (first) g_main_context_invoke(NULL, quit_loop, run);
}
static gboolean force_key_frame(gpointer data) {
CaptureRun *run = data;
if (run->encoder == NULL) return G_SOURCE_REMOVE;
GstPad *source = gst_element_get_static_pad(run->encoder, "src");
if (source != NULL) {
gst_pad_send_event(
source,
gst_video_event_new_upstream_force_key_unit(GST_CLOCK_TIME_NONE, TRUE,
0));
gst_object_unref(source);
static void fail_output(VideoOutput *output, const char *message) {
write_frame(6, 0, (guint8)output->layer, 0, 0, 0, 0,
(const guint8 *)message, MIN(strlen(message), 512));
fail_run(output->run, message);
}
static gboolean set_output_active(VideoOutput *output, gboolean enabled) {
CaptureRun *run = output->run;
g_mutex_lock(&run->lock);
if (output->enabled == enabled) {
g_mutex_unlock(&run->lock);
return TRUE;
}
return G_SOURCE_REMOVE;
output->enabled = FALSE;
output->decodable = FALSE;
output->activation_timestamp = GST_CLOCK_TIME_NONE;
output->key_timestamp = GST_CLOCK_TIME_NONE;
++output->key_requested;
g_mutex_unlock(&run->lock);
// A closed valve isolates the retired branch from the other tee subscribers.
g_object_set(output->gate, "drop", TRUE, NULL);
gst_element_set_locked_state(output->branch, !enabled);
gboolean applied = enabled
? gst_element_sync_state_with_parent(output->branch)
: gst_element_set_state(output->branch, GST_STATE_NULL) != GST_STATE_CHANGE_FAILURE;
if (!applied) return FALSE;
g_mutex_lock(&run->lock);
output->enabled = enabled;
g_mutex_unlock(&run->lock);
if (enabled) g_object_set(output->gate, "drop", FALSE, NULL);
return TRUE;
}
static gpointer control_input(gpointer data) {
static gboolean apply_control(CaptureRun *run, char *line) {
char *values[4] = {0};
guint count = 0;
g_auto(GStrv) tokens = g_strsplit_set(line, " \t\r", -1);
for (guint index = 0; tokens[index] != NULL; ++index) {
char *part = tokens[index];
if (part[0] == '\0') continue;
if (count == G_N_ELEMENTS(values)) return FALSE;
values[count++] = part;
}
if (count == 1 && strcmp(values[0], "Q") == 0) {
g_atomic_int_set(&run->stopping, TRUE);
g_main_loop_quit(run->loop);
return TRUE;
}
if (run->output_count == 0) return FALSE;
guint layer = 0;
if (count == 2 && strcmp(values[0], "K") == 0) {
gboolean all = strcmp(values[1], "-1") == 0;
if (!all && !parse_uint(values[1], 0, run->output_count - 1, &layer)) return FALSE;
g_mutex_lock(&run->lock);
for (guint index = 0; index < run->output_count; ++index) {
if (all || index == layer) ++run->outputs[index].key_requested;
}
g_mutex_unlock(&run->lock);
return TRUE;
}
if (count == 3 && strcmp(values[0], "A") == 0) {
guint active = 0;
if (!parse_uint(values[1], 0, run->output_count - 1, &layer) ||
!parse_uint(values[2], 0, 1, &active)) return FALSE;
return set_output_active(&run->outputs[layer], active != 0);
}
if (count == 3 && strcmp(values[0], "B") == 0 &&
parse_uint(values[1], 0, run->output_count - 1, &layer)) {
VideoOutput *output = &run->outputs[layer];
guint bitrate = 0;
if (!parse_uint(values[2], 1000, output->profile.bit_rate, &bitrate)) return FALSE;
GParamSpec *property = g_object_class_find_property(
G_OBJECT_GET_CLASS(output->encoder), "bitrate");
if (property == NULL || !(property->flags & GST_PARAM_MUTABLE_PLAYING) ||
!set_numeric_property(G_OBJECT(output->encoder), "bitrate",
MAX(1, bitrate / run->encoder_info->bit_rate_divisor))) {
fail_output(output, "hardware encoder cannot apply the live bitrate");
}
return TRUE;
}
return FALSE;
}
static gboolean valid_input_header(CaptureRun *run) {
const guint8 *header = run->input_header;
if (memcmp(header, "SMED", 4) != 0 || header[4] != 2) return FALSE;
run->input_size = (guint)read_be(header + 28, 4);
if (header[5] == 7) {
for (guint index = 6; index < 28; ++index) if (header[index] != 0) return FALSE;
return run->input_size > 0 && run->input_size <= 64;
}
if (header[5] != 2 || run->encoded_source == NULL || header[6] > 1 || header[7] != 0 ||
run->input_size == 0 || run->input_size > kMaxPayloadBytes) return FALSE;
guint width = (guint)read_be(header + 24, 2), height = (guint)read_be(header + 26, 2);
guint64 timestamp = read_be(header + 8, 8), duration = read_be(header + 16, 8);
return width >= 2 && width <= 2560 && height >= 2 && height <= 1440 &&
width % 2 == 0 && height % 2 == 0 && duration > 0 &&
timestamp <= INT64_MAX / 100 && duration <= INT64_MAX / 100;
}
static gboolean apply_input(CaptureRun *run) {
const guint8 *payload = run->input_payload->data;
if (run->input_header[5] == 7) {
for (guint index = 0; index < run->input_size; ++index) if (payload[index] < 32 || payload[index] > 126) return FALSE;
char command[65];
memcpy(command, payload, run->input_size);
command[run->input_size] = '\0';
return apply_control(run, command);
}
NalSummary nal = inspect_h264(payload, run->input_size);
guint64 timestamp = read_be(run->input_header + 8, 8);
if ((!run->encoded_started && !(nal.sps && nal.pps && nal.idr)) ||
(run->encoded_started && timestamp <= run->encoded_timestamp)) return FALSE;
GstBuffer *buffer = gst_buffer_new_allocate(NULL, run->input_size, NULL);
if (buffer == NULL) return FALSE;
gst_buffer_fill(buffer, 0, payload, run->input_size);
GST_BUFFER_PTS(buffer) = timestamp * 100;
GST_BUFFER_DTS(buffer) = GST_CLOCK_TIME_NONE;
GST_BUFFER_DURATION(buffer) = read_be(run->input_header + 16, 8) * 100;
if (!nal.idr) GST_BUFFER_FLAG_SET(buffer, GST_BUFFER_FLAG_DELTA_UNIT);
run->encoded_started = TRUE;
run->encoded_timestamp = timestamp;
// appsrc blocks at one queued AU; compressed reference frames are never dropped.
return gst_app_src_push_buffer(GST_APP_SRC(run->encoded_source), buffer) == GST_FLOW_OK;
}
static gboolean control_input(gint fd, GIOCondition condition, gpointer data) {
CaptureRun *run = data;
guint8 value = 0;
while (read(STDIN_FILENO, &value, 1) == 1) {
if (value == 'K') {
g_main_context_invoke(NULL, force_key_frame, run);
} else if (value == 'Q' || value == '\n' || value == '\r') {
g_atomic_int_set(&run->stopping, TRUE);
g_main_context_invoke(NULL, quit_loop, run);
return NULL;
guint8 bytes[32 * 1024];
ssize_t size = read(fd, bytes, sizeof(bytes));
if (size < 0 && errno == EINTR) return G_SOURCE_CONTINUE;
if (size <= 0) {
g_atomic_int_set(&run->stopping, TRUE);
g_main_loop_quit(run->loop);
return G_SOURCE_CONTINUE;
}
(void)condition;
gsize offset = 0;
while (offset < (gsize)size && !g_atomic_int_get(&run->stopping)) {
if (run->input_header_length < 32) {
gsize take = MIN(32 - run->input_header_length, (gsize)size - offset);
memcpy(run->input_header + run->input_header_length, bytes + offset, take);
offset += take;
run->input_header_length += (guint)take;
if (run->input_header_length < 32) continue;
if (!valid_input_header(run)) {
fail_run(run, "invalid native input envelope");
return G_SOURCE_CONTINUE;
}
}
gsize take = MIN(run->input_size - run->input_payload->len, (gsize)size - offset);
g_byte_array_append(run->input_payload, bytes + offset, (guint)take);
offset += take;
if (run->input_payload->len < run->input_size) continue;
if (!apply_input(run)) {
fail_run(run, "invalid or failed native input");
return G_SOURCE_CONTINUE;
}
run->input_header_length = 0;
run->input_size = 0;
g_byte_array_set_size(run->input_payload, 0);
}
return G_SOURCE_CONTINUE;
}
static GstPadProbeReturn begin_input_frame(GstPad *pad, GstPadProbeInfo *info,
gpointer data) {
(void)pad;
CaptureRun *run = data;
GstBuffer *buffer = GST_PAD_PROBE_INFO_BUFFER(info);
if (buffer == NULL || !GST_BUFFER_PTS_IS_VALID(buffer)) {
fail_run(run, "capture input has no presentation timestamp");
return GST_PAD_PROBE_DROP;
}
GstClockTime timestamp = GST_BUFFER_PTS(buffer);
if (GST_CLOCK_TIME_IS_VALID(run->last_input_timestamp) &&
timestamp <= run->last_input_timestamp) return GST_PAD_PROBE_DROP;
run->last_input_timestamp = timestamp;
GstClockTime duration = GST_BUFFER_DURATION_IS_VALID(buffer)
? GST_BUFFER_DURATION(buffer)
: GST_SECOND / run->profile.frame_rate;
if (!write_frame(5, 0, 0, 0, 0, timestamp / 100, MAX(1, duration / 100), NULL, 0)) {
fail_run(run, "native media output closed");
return GST_PAD_PROBE_DROP;
}
return GST_PAD_PROBE_OK;
}
static GstPadProbeReturn output_input_frame(GstPad *pad, GstPadProbeInfo *info,
gpointer data) {
VideoOutput *output = data;
CaptureRun *run = output->run;
GstBuffer *buffer = GST_PAD_PROBE_INFO_BUFFER(info);
if (buffer == NULL || !GST_BUFFER_PTS_IS_VALID(buffer)) return GST_PAD_PROBE_DROP;
GstClockTime timestamp = GST_BUFFER_PTS(buffer);
g_mutex_lock(&run->lock);
gboolean enabled = output->enabled;
gboolean force = enabled && output->key_requested != output->key_sent;
guint64 request = output->key_requested;
if (enabled && !GST_CLOCK_TIME_IS_VALID(output->activation_timestamp)) {
output->activation_timestamp = timestamp;
}
if (force) {
output->key_sent = request;
output->key_timestamp = timestamp;
}
g_mutex_unlock(&run->lock);
if (!enabled) return GST_PAD_PROBE_DROP;
if (force) {
gboolean sent = gst_pad_push_event(pad,
gst_video_event_new_downstream_force_key_unit(
timestamp, GST_CLOCK_TIME_NONE, GST_CLOCK_TIME_NONE, TRUE,
(guint)request));
if (!sent) {
g_mutex_lock(&run->lock);
if (output->key_sent == request) output->key_sent = output->key_acknowledged;
g_mutex_unlock(&run->lock);
}
}
g_atomic_int_set(&run->stopping, TRUE);
g_main_context_invoke(NULL, quit_loop, run);
return NULL;
return GST_PAD_PROBE_OK;
}
static gboolean bus_message(GstBus *bus, GstMessage *message, gpointer data) {
@@ -523,6 +837,12 @@ static gboolean bus_message(GstBus *bus, GstMessage *message, gpointer data) {
GError *error = NULL;
char *debug = NULL;
gst_message_parse_error(message, &error, &debug);
for (guint index = 0; index < run->output_count; ++index) {
if (GST_MESSAGE_SRC(message) == GST_OBJECT(run->outputs[index].encoder)) {
fail_output(&run->outputs[index], "hardware encoder output failed");
break;
}
}
fail_run(run, error == NULL ? "GStreamer pipeline failed" : error->message);
g_clear_error(&error);
g_free(debug);
@@ -534,7 +854,8 @@ static gboolean bus_message(GstBus *bus, GstMessage *message, gpointer data) {
}
static GstFlowReturn video_sample(GstAppSink *sink, gpointer data) {
CaptureRun *run = data;
VideoOutput *output = data;
CaptureRun *run = output->run;
GstSample *sample = gst_app_sink_pull_sample(sink);
if (sample == NULL) return GST_FLOW_EOS;
GstBuffer *buffer = gst_sample_get_buffer(sample);
@@ -543,30 +864,44 @@ static GstFlowReturn video_sample(GstAppSink *sink, gpointer data) {
mapped.size == 0 || mapped.size > kMaxPayloadBytes) {
if (buffer != NULL && mapped.data != NULL) gst_buffer_unmap(buffer, &mapped);
gst_sample_unref(sample);
fail_run(run, "GStreamer returned an invalid H.264 access unit");
fail_output(output, "GStreamer returned an invalid H.264 access unit");
return GST_FLOW_ERROR;
}
gboolean key_frame = !GST_BUFFER_FLAG_IS_SET(buffer, GST_BUFFER_FLAG_DELTA_UNIT);
NalSummary nal = inspect_h264(mapped.data, mapped.size);
if (!run->active &&
(!key_frame || !nal.sps || !nal.pps || !nal.idr ||
nal.profile_level_id[0] == '\0')) {
if (!GST_BUFFER_PTS_IS_VALID(buffer)) {
gst_buffer_unmap(buffer, &mapped);
gst_sample_unref(sample);
fail_output(output, "encoded output lost the source presentation timestamp");
return GST_FLOW_ERROR;
}
GstClockTime timestamp = GST_BUFFER_PTS(buffer);
gboolean recovery = key_frame && nal.sps && nal.pps && nal.idr &&
nal.profile_level_id[0] != '\0';
g_mutex_lock(&run->lock);
gboolean current = output->enabled &&
GST_CLOCK_TIME_IS_VALID(output->activation_timestamp) &&
timestamp >= output->activation_timestamp;
if (current && GST_CLOCK_TIME_IS_VALID(output->key_timestamp) &&
timestamp >= output->key_timestamp) {
if (recovery) output->key_acknowledged = output->key_sent;
else output->key_sent = output->key_acknowledged;
}
if (current && recovery) output->decodable = TRUE;
gboolean deliver = current && output->decodable;
gboolean publish_active = deliver && !run->active &&
output->layer == run->original_output;
if (publish_active) run->active = TRUE;
g_mutex_unlock(&run->lock);
if (!deliver) {
gst_buffer_unmap(buffer, &mapped);
gst_sample_unref(sample);
g_main_context_invoke(NULL, force_key_frame, run);
return GST_FLOW_OK;
}
guint64 duration = GST_BUFFER_DURATION_IS_VALID(buffer)
? GST_BUFFER_DURATION(buffer) / 100
: 10000000ULL / run->profile.frame_rate;
guint64 timestamp = GST_BUFFER_PTS_IS_VALID(buffer)
? GST_BUFFER_PTS(buffer) / 100
: run->last_timestamp + duration;
if (run->last_timestamp != 0 && timestamp <= run->last_timestamp) {
timestamp = run->last_timestamp + duration;
}
run->last_timestamp = timestamp;
gboolean written = write_frame(2, key_frame ? 1 : 0, timestamp, duration,
gboolean written = write_frame(2, recovery ? 1 : 0, (guint8)output->layer,
output->profile.width, output->profile.height,
timestamp / 100,
10000000ULL / output->profile.frame_rate,
mapped.data, mapped.size);
gst_buffer_unmap(buffer, &mapped);
gst_sample_unref(sample);
@@ -574,18 +909,20 @@ static GstFlowReturn video_sample(GstAppSink *sink, gpointer data) {
fail_run(run, "native media output closed");
return GST_FLOW_ERROR;
}
if (!run->active) {
if (publish_active) {
char *token = json_string(run->restore_token);
char *profiles = output_profiles_json(run);
char *status = g_strdup_printf(
"{\"state\":\"active\",\"codec\":\"h264\",\"hardwareOnly\":true,"
"\"profileLevelId\":\"%s\",\"width\":%u,\"height\":%u,"
"\"fps\":%u,\"restoreToken\":%s}",
"\"fps\":%u,\"restoreToken\":%s,\"outputs\":%s}",
nal.profile_level_id, run->profile.width, run->profile.height,
run->profile.frame_rate, token);
run->profile.frame_rate, token, profiles);
g_free(token);
run->active = write_status(status);
g_free(profiles);
gboolean active_written = write_status(status);
g_free(status);
if (!run->active) {
if (!active_written) {
fail_run(run, "native media output closed");
return GST_FLOW_ERROR;
}
@@ -620,7 +957,7 @@ static GstFlowReturn audio_sample(GstAppSink *sink, gpointer data) {
gst_buffer_unmap(buffer, &mapped);
gst_sample_unref(sample);
while (run->audio->len >= kAudioFrameBytes) {
if (!write_frame(1, 0, run->audio_timestamp, 200000,
if (!write_frame(1, 0, 0, 0, 0, run->audio_timestamp, 200000,
run->audio->data, kAudioFrameBytes)) {
fail_run(run, "native audio output closed");
return GST_FLOW_ERROR;
@@ -631,27 +968,34 @@ static GstFlowReturn audio_sample(GstAppSink *sink, gpointer data) {
return GST_FLOW_OK;
}
static int run_pipeline(CaptureRun *run, GCallback sample_callback,
const char *starting_status) {
static int run_pipeline(CaptureRun *run, const char *starting_status) {
run->input_payload = g_byte_array_new();
GstBus *bus = gst_element_get_bus(run->pipeline);
guint bus_watch = gst_bus_add_watch(bus, bus_message, run);
gst_object_unref(bus);
GstElement *output = gst_bin_get_by_name(GST_BIN(run->pipeline), "output");
if (output == NULL) return 2;
g_signal_connect(output, "new-sample", sample_callback, run);
gst_object_unref(output);
if (starting_status != NULL && !write_status(starting_status)) return 2;
guint control_watch = g_unix_fd_add(STDIN_FILENO, G_IO_IN | G_IO_HUP | G_IO_ERR,
control_input, run);
if (starting_status != NULL && !write_status(starting_status)) {
fail_run(run, "native media output closed");
goto stopped;
}
GstStateChangeReturn state =
gst_element_set_state(run->pipeline, GST_STATE_PLAYING);
if (state == GST_STATE_CHANGE_FAILURE) return 2;
if (state == GST_STATE_CHANGE_FAILURE) {
fail_run(run, "GStreamer pipeline failed to start");
goto stopped;
}
if (gst_element_get_state(run->pipeline, NULL, NULL, 5 * GST_SECOND) ==
GST_STATE_CHANGE_FAILURE) {
return 2;
fail_run(run, "GStreamer pipeline failed to start");
goto stopped;
}
g_thread_unref(g_thread_new("screener-control", control_input, run));
g_main_loop_run(run->loop);
stopped:
g_source_remove(control_watch);
gst_element_set_state(run->pipeline, GST_STATE_NULL);
g_source_remove(bus_watch);
g_byte_array_unref(run->input_payload);
g_mutex_lock(&run->lock);
char *failure = g_strdup(run->failure);
g_mutex_unlock(&run->lock);
@@ -664,18 +1008,22 @@ static int run_pipeline(CaptureRun *run, GCallback sample_callback,
}
static int capture_video(int count, char **values) {
VideoProfile profile = {0};
CaptureRun run = {.last_input_timestamp = GST_CLOCK_TIME_NONE};
gboolean encoded = count > 1 && strcmp(values[1], "--encoded-video") == 0;
guint encoder_index = 0;
if (!parse_profile(count, values, &profile, &encoder_index)) return 2;
if (!parse_profile(count, values, &run, &encoder_index)) return 2;
GPtrArray *encoders = hardware_encoders();
if (!video_stack_available() || encoder_index >= encoders->len) {
gboolean input_available = encoded
? has_factory("appsrc") && has_factory("h264parse") && has_factory("decodebin")
: video_stack_available();
if (!input_available || encoder_index >= encoders->len) {
g_ptr_array_unref(encoders);
return 2;
}
const EncoderInfo *encoder_info = g_ptr_array_index(encoders, encoder_index);
GError *error = NULL;
ScreenerPortalCapture portal = {.pipewire_fd = -1};
if (!screener_portal_capture_open(
if (!encoded && !screener_portal_capture_open(
g_getenv("SCREENER_XDP_RESTORE_TOKEN"), &portal, &error)) {
fprintf(stderr, "Screener capture unavailable: %s\n",
error == NULL ? "screen selection failed" : error->message);
@@ -684,19 +1032,36 @@ static int capture_video(int count, char **values) {
return 2;
}
char *pipeline_text = g_strdup_printf(
GString *pipeline_text = g_string_new(NULL);
if (encoded) {
g_string_append(pipeline_text,
"appsrc name=source is-live=true format=time do-timestamp=false block=true "
"max-buffers=1 max-bytes=1048576 max-time=0 "
"caps=video/x-h264,stream-format=byte-stream,alignment=au ! "
"h264parse ! decodebin ! video/x-raw ! tee name=frames ");
} else g_string_append_printf(pipeline_text,
"pipewiresrc name=source fd=%d target-object=%s do-timestamp=true ! "
"queue max-size-buffers=2 max-size-bytes=0 max-size-time=0 "
"leaky=downstream ! videoconvert ! videoscale ! videorate ! "
"video/x-raw,format=NV12,width=%u,height=%u,framerate=%u/1 ! "
"%s name=encoder ! video/x-h264,profile=constrained-baseline ! "
"h264parse config-interval=-1 ! "
"video/x-h264,stream-format=byte-stream,alignment=au ! "
"appsink name=output emit-signals=true sync=false max-buffers=2 drop=true",
portal.pipewire_fd, portal.target_object, profile.width, profile.height,
profile.frame_rate, encoder_info->factory);
GstElement *pipeline = gst_parse_launch(pipeline_text, &error);
g_free(pipeline_text);
"videorate drop-only=true ! video/x-raw,framerate=%u/1 ! tee name=frames ",
portal.pipewire_fd, portal.target_object, run.profile.frame_rate);
for (guint index = 0; index < run.output_count; ++index) {
const VideoProfile *profile = &run.outputs[index].profile;
g_string_append_printf(pipeline_text,
"frames. ! valve name=gate%u drop=%s ! ( "
"queue name=raw%u max-size-buffers=1 max-size-bytes=0 "
"max-size-time=0 leaky=downstream ! videorate drop-only=true ! "
"video/x-raw,framerate=%u/1 ! videoconvert ! videoscale ! "
"video/x-raw,format=NV12,width=%u,height=%u ! "
"%s name=encoder%u ! video/x-h264,profile=constrained-baseline ! "
"h264parse config-interval=-1 ! "
"video/x-h264,stream-format=byte-stream,alignment=au ! "
"appsink name=output%u emit-signals=true sync=false async=false "
"max-buffers=1 drop=false ) ",
index, run.outputs[index].enabled ? "false" : "true",
index, profile->frame_rate, profile->width, profile->height, encoder_info->factory,
index, index);
}
GstElement *pipeline = gst_parse_launch(pipeline_text->str, &error);
g_string_free(pipeline_text, TRUE);
if (pipeline == NULL || error != NULL) {
fprintf(stderr, "Screener capture unavailable: %s\n",
error == NULL ? "video pipeline failed" : error->message);
@@ -706,48 +1071,85 @@ static int capture_video(int count, char **values) {
g_ptr_array_unref(encoders);
return 2;
}
CaptureRun run = {
.loop = g_main_loop_new(NULL, FALSE),
.pipeline = pipeline,
.profile = profile,
.encoder_info = encoder_info,
.restore_token = portal.restore_token,
};
run.loop = g_main_loop_new(NULL, FALSE);
run.pipeline = pipeline;
run.encoder_info = encoder_info;
run.restore_token = portal.restore_token;
g_mutex_init(&run.lock);
run.encoder = gst_bin_get_by_name(GST_BIN(pipeline), "encoder");
int result = 2;
GstElement *source = gst_bin_get_by_name(GST_BIN(pipeline), "source");
if (source != NULL) {
set_optional_boolean(G_OBJECT(source), "resend-last", TRUE);
set_numeric_property(G_OBJECT(source), "keepalive-time",
MAX(1, 1000 / profile.frame_rate));
gst_object_unref(source);
if (encoded) run.encoded_source = source;
else {
set_optional_boolean(G_OBJECT(source), "resend-last", TRUE);
set_numeric_property(G_OBJECT(source), "keepalive-time",
MAX(1, 1000 / run.profile.frame_rate));
gst_object_unref(source);
}
}
if (run.encoder == NULL || !configure_encoder(&run)) {
fprintf(stderr, "Screener capture unavailable: hardware encoder is incompatible\n");
if (run.encoder != NULL) gst_object_unref(run.encoder);
g_main_loop_unref(run.loop);
g_mutex_clear(&run.lock);
gst_object_unref(pipeline);
screener_portal_capture_close(&portal);
g_ptr_array_unref(encoders);
return 2;
GstElement *tee = gst_bin_get_by_name(GST_BIN(pipeline), "frames");
GstPad *input = tee == NULL ? NULL : gst_element_get_static_pad(tee, "sink");
if (tee != NULL) gst_object_unref(tee);
if (input == NULL) goto cleanup;
gst_pad_add_probe(input, GST_PAD_PROBE_TYPE_BUFFER, begin_input_frame, &run, NULL);
gst_object_unref(input);
for (guint index = 0; index < run.output_count; ++index) {
VideoOutput *output = &run.outputs[index];
char *name = g_strdup_printf("encoder%u", index);
output->encoder = gst_bin_get_by_name(GST_BIN(pipeline), name);
g_free(name);
if (output->encoder == NULL || !configure_encoder(output)) {
fprintf(stderr, "Screener capture unavailable: hardware encoder is incompatible\n");
write_frame(6, 0, (guint8)index, 0, 0, 0, 0,
(const guint8 *)"hardware encoder configuration failed",
strlen("hardware encoder configuration failed"));
goto cleanup;
}
output->branch = GST_ELEMENT(gst_object_get_parent(GST_OBJECT(output->encoder)));
name = g_strdup_printf("gate%u", index);
output->gate = gst_bin_get_by_name(GST_BIN(pipeline), name);
g_free(name);
if (output->branch == NULL || output->branch == pipeline || output->gate == NULL) goto cleanup;
gst_element_set_locked_state(output->branch, !output->enabled);
name = g_strdup_printf("output%u", index);
GstElement *sink = gst_bin_get_by_name(GST_BIN(pipeline), name);
g_free(name);
if (sink == NULL) goto cleanup;
g_signal_connect(sink, "new-sample", G_CALLBACK(video_sample), output);
gst_object_unref(sink);
name = g_strdup_printf("raw%u", index);
GstElement *queue = gst_bin_get_by_name(GST_BIN(pipeline), name);
g_free(name);
GstPad *raw = queue == NULL ? NULL : gst_element_get_static_pad(queue, "src");
if (queue != NULL) gst_object_unref(queue);
if (raw == NULL) goto cleanup;
gst_pad_add_probe(raw, GST_PAD_PROBE_TYPE_BUFFER, output_input_frame, output, NULL);
gst_object_unref(raw);
}
char *adapter_name = json_string("GStreamer hardware H.264");
char *encoder_name = json_string(encoder_info->name);
char *encoder_identity = json_string(encoder_info->factory);
char *profiles = output_profiles_json(&run);
char *starting = g_strdup_printf(
"{\"state\":\"starting\",\"codec\":\"h264\",\"hardwareOnly\":true,"
"\"adapterIndex\":0,\"adapterName\":%s,"
"\"adapterIdentity\":\"gstreamer-hardware-h264\","
"\"encoderIndex\":%u,\"encoderName\":%s,"
"\"encoderIdentity\":%s}",
adapter_name, encoder_index, encoder_name, encoder_identity);
"\"encoderIdentity\":%s,\"outputs\":%s}",
adapter_name, encoder_index, encoder_name, encoder_identity, profiles);
g_free(adapter_name);
g_free(encoder_name);
g_free(encoder_identity);
int result = run_pipeline(&run, G_CALLBACK(video_sample), starting);
g_free(profiles);
result = run_pipeline(&run, starting);
g_free(starting);
gst_object_unref(run.encoder);
cleanup:
if (run.encoded_source != NULL) gst_object_unref(run.encoded_source);
for (guint index = 0; index < run.output_count; ++index) {
if (run.outputs[index].encoder != NULL) gst_object_unref(run.outputs[index].encoder);
if (run.outputs[index].branch != NULL) gst_object_unref(run.outputs[index].branch);
if (run.outputs[index].gate != NULL) gst_object_unref(run.outputs[index].gate);
}
g_free(run.failure);
g_main_loop_unref(run.loop);
g_mutex_clear(&run.lock);
@@ -784,7 +1186,12 @@ static int capture_audio(int count, char **values) {
.audio = g_byte_array_new(),
};
g_mutex_init(&run.lock);
int result = run_pipeline(&run, G_CALLBACK(audio_sample), NULL);
GstElement *output = gst_bin_get_by_name(GST_BIN(pipeline), "output");
if (output != NULL) {
g_signal_connect(output, "new-sample", G_CALLBACK(audio_sample), &run);
gst_object_unref(output);
}
int result = output == NULL ? 2 : run_pipeline(&run, NULL);
g_byte_array_unref(run.audio);
g_free(run.failure);
g_main_loop_unref(run.loop);
@@ -797,7 +1204,7 @@ int main(int argc, char **argv) {
gst_init(&argc, &argv);
if (argc == 2 && strcmp(argv[1], "--probe") == 0) return write_probe();
if (argc == 2 && strcmp(argv[1], "--list") == 0) return write_sources();
if (argc > 1 && strcmp(argv[1], "--capture-video") == 0) {
if (argc > 1 && (strcmp(argv[1], "--capture-video") == 0 || strcmp(argv[1], "--encoded-video") == 0)) {
return capture_video(argc, argv);
}
if (argc > 1 && strcmp(argv[1], "--capture-audio") == 0) {
native/capture/linux/main.test.c
+118
View File
@@ -0,0 +1,118 @@
// Build with portal.c and the same pkg-config flags as build.sh; no capture or codec is used.
#define main capture_main
#include "main.c"
#undef main
static void check_slot_profiles(void) {
char *arguments[23 + 7 * 5] = {
"capture", "--capture-video", "picker", "1", "0", "0",
"--adapter-index", "0", "--mft-index", "0", "--width", "1280",
"--height", "720", "--fps", "30", "--bitrate", "3000000",
"--preference", "balanced", "--codec", "h264", "--protocol-v7"};
for (guint index = 0; index < 7; ++index) {
guint offset = 23 + index * 5;
arguments[offset] = "--output";
arguments[offset + 1] = index == 1 ? "1280" : "854";
arguments[offset + 2] = index == 1 ? "720" : "480";
arguments[offset + 3] = "30";
arguments[offset + 4] = index == 1 ? "3000000" : "2000000";
}
CaptureRun run = {0};
guint encoder = 0;
g_assert_true(parse_profile(53, arguments, &run, &encoder));
g_assert_cmpuint(run.output_count, ==, 6);
g_assert_cmpuint(run.original_output, ==, 1);
for (guint index = 0; index < run.output_count; ++index) {
g_assert_cmpint(run.outputs[index].enabled, ==, index < 2);
}
g_assert_false(parse_profile(58, arguments, &run, &encoder));
arguments[23 + 5 * 5 + 1] = "1920";
g_assert_false(parse_profile(53, arguments, &run, &encoder));
arguments[23 + 5 * 5 + 1] = "854";
arguments[23 + 5 + 1] = "854";
g_assert_false(parse_profile(53, arguments, &run, &encoder));
arguments[23 + 5 + 1] = "1280";
char *encoded[11 + 6 * 5] = {
"capture", "--encoded-video", "--codec", "h264", "--adapter-index", "0",
"--mft-index", "0", "--preference", "balanced", "--protocol-v7"};
memcpy(encoded + 11, arguments + 23, 6 * 5 * sizeof(char *));
run = (CaptureRun){0};
g_assert_true(parse_profile(G_N_ELEMENTS(encoded), encoded, &run, &encoder));
g_assert_cmpuint(run.profile.width, ==, 1280);
g_assert_cmpuint(run.profile.height, ==, 720);
g_assert_cmpuint(run.original_output, ==, 0);
for (guint index = 0; index < run.output_count; ++index) {
g_assert_false(run.outputs[index].enabled);
}
}
static GstState branch_state(VideoOutput *output) {
GstState state = GST_STATE_VOID_PENDING;
g_assert_cmpint(gst_element_get_state(output->branch, &state, NULL, GST_SECOND),
!=, GST_STATE_CHANGE_FAILURE);
return state;
}
static void check_slot_retirement(void) {
CaptureRun run = {.output_count = 6};
g_mutex_init(&run.lock);
GString *description = g_string_new("fakesrc is-live=true num-buffers=8 ! tee name=frames ");
for (guint index = 0; index < run.output_count; ++index) {
g_string_append_printf(description,
"frames. ! valve name=gate%u drop=true ! ( identity name=encoder%u ! "
"fakesink sync=false async=false ) ", index, index);
}
GError *error = NULL;
run.pipeline = gst_parse_launch(description->str, &error);
g_string_free(description, TRUE);
g_assert_no_error(error);
g_assert_nonnull(run.pipeline);
for (guint index = 0; index < run.output_count; ++index) {
VideoOutput *output = &run.outputs[index];
output->run = &run;
output->profile.bit_rate = 2000000;
char *name = g_strdup_printf("encoder%u", index);
output->encoder = gst_bin_get_by_name(GST_BIN(run.pipeline), name);
g_free(name);
g_assert_nonnull(output->encoder);
output->branch = GST_ELEMENT(gst_object_get_parent(GST_OBJECT(output->encoder)));
g_assert_true(output->branch != run.pipeline);
name = g_strdup_printf("gate%u", index);
output->gate = gst_bin_get_by_name(GST_BIN(run.pipeline), name);
g_free(name);
g_assert_nonnull(output->gate);
gst_element_set_locked_state(output->branch, TRUE);
}
g_assert_cmpint(gst_element_set_state(run.pipeline, GST_STATE_PLAYING), !=, GST_STATE_CHANGE_FAILURE);
g_assert_true(apply_control(&run, "A 1 1"));
g_assert_true(apply_control(&run, "A 5 1"));
g_assert_cmpint(branch_state(&run.outputs[1]), ==, GST_STATE_PLAYING);
g_assert_cmpint(branch_state(&run.outputs[5]), ==, GST_STATE_PLAYING);
g_assert_cmpint(branch_state(&run.outputs[2]), ==, GST_STATE_NULL);
g_assert_true(apply_control(&run, "A 1 0"));
g_assert_cmpint(branch_state(&run.outputs[1]), ==, GST_STATE_NULL);
g_assert_cmpint(branch_state(&run.outputs[5]), ==, GST_STATE_PLAYING);
g_assert_true(apply_control(&run, "A 1 1"));
g_assert_cmpint(branch_state(&run.outputs[1]), ==, GST_STATE_PLAYING);
g_assert_true(apply_control(&run, "K 5"));
g_assert_false(apply_control(&run, "A 2"));
g_assert_false(apply_control(&run, "A 6 1"));
g_assert_false(apply_control(&run, "A 0 2"));
g_assert_false(apply_control(&run, "B 5 999"));
gst_element_set_state(run.pipeline, GST_STATE_NULL);
for (guint index = 0; index < run.output_count; ++index) {
gst_object_unref(run.outputs[index].encoder);
gst_object_unref(run.outputs[index].branch);
gst_object_unref(run.outputs[index].gate);
}
gst_object_unref(run.pipeline);
g_mutex_clear(&run.lock);
}
int main(int argc, char **argv) {
gst_init(&argc, &argv);
check_slot_profiles();
check_slot_retirement();
return 0;
}
native/capture/windows/README.md
+13 -4
View File
@@ -27,8 +27,8 @@ codec remains fixed across profile and source changes. VP8 reads the existing
NV12 surface through one staging texture and uses the same encoded-frame
boundary. The process has no network fallback. The Client
consumes the selected process or system-audio stream through its native media
edge when the capability probe reports support. Build it outside the repository
for a bounded capability run:
edge when the capability probe reports support. Build to the stable project
`build/client-check` directory for a bounded capability run:
```powershell
npm run check:client-capture
@@ -38,8 +38,17 @@ The implementation follows Microsoft's MIT-licensed reference samples and
official API contracts without copying their WIL framework. The retained MF
fixture compiles the same encoder source with `SCREENER_H264_FIXTURE`; there is
not a second product MFT implementation.
The build caches pinned libvpx, NASM and make inputs under `build/client-check`;
only statically linked VP8 code and its notices enter the package.
Shared local outputs use stock WebRTC source/encoder adaptation. H264 keeps
native NV12 surfaces, while WebRTC VP8 reads back into I420. The existing capture
and output mailboxes retain source/generation ownership; adapted sizes and
timestamps cross the same media envelope. Hardware low-latency mode stays on;
the old fixed one-frame VBV override is removed so later bitrate increases are
not trapped by the original buffer limit.
The build caches the pinned WebRTC SDK and matching official MSVC linker/runtime
libraries under `build/encoder-pool`. The SDK supplies libvpx, replacing the
separate source build. Only statically linked code and required notices enter
the package, not the SDK/toolchain or another runtime service.
- <https://github.com/microsoft/Windows-classic-samples/tree/main/Samples/ApplicationLoopback>
- <https://github.com/microsoft/Windows.UI.Composition-Win32-Samples/tree/master/cpp/ScreenCaptureforHWND>
native/capture/windows/adaptive_encoder.cpp
+600
View File
@@ -0,0 +1,600 @@
#include "adaptive_encoder.h"
#include "capture_output.h"
#include <algorithm>
#include <cmath>
#include <exception>
#include <future>
#include <limits>
#include <mutex>
#include <thread>
#include "api/environment/environment_factory.h"
#include "api/make_ref_counted.h"
#include "api/task_queue/task_queue_base.h"
#include "api/task_queue/task_queue_factory.h"
#include "api/video/builtin_video_bitrate_allocator_factory.h"
#include "api/video/i420_buffer.h"
#include "api/video/video_adapter.h"
#include "api/video/video_frame.h"
#include "api/video/video_stream_encoder_settings.h"
#include "api/video_codecs/builtin_video_encoder_factory.h"
#include "api/video_codecs/video_encoder_factory.h"
#include "call/video_send_stream.h"
#include "libyuv/convert.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "system_wrappers/include/clock.h"
#include "video/adaptation/overuse_frame_detector.h"
#include "video/frame_cadence_adapter.h"
#include "video/send_statistics_proxy.h"
#include "video/video_stream_encoder.h"
namespace screener::capture::windows {
namespace {
template <typename F>
void Sync(webrtc::TaskQueueBase* queue, F fn) {
std::promise<void> completed;
auto result = completed.get_future();
queue->PostTask([completed = std::move(completed), fn = std::move(fn)]() mutable {
try {
fn();
completed.set_value();
} catch (...) {
completed.set_exception(std::current_exception());
}
});
result.get();
}
// Library callbacks cannot throw through WebRTC's task queues.
class Failure final {
public:
void Capture() noexcept {
std::lock_guard<std::mutex> lock(mutex_);
if (!failure_) failure_ = std::current_exception();
}
void Set(const char* detail) noexcept {
try {
throw GateFailure("adaptive-encoder", detail);
} catch (...) {
Capture();
}
}
void Rethrow() {
std::exception_ptr failure;
{
std::lock_guard<std::mutex> lock(mutex_);
failure = failure_;
}
if (failure) {
try {
std::rethrow_exception(failure);
} catch (const GateFailure&) {
throw;
} catch (const std::exception& error) {
throw GateFailure("adaptive-encoder", error.what());
} catch (...) {
throw GateFailure("adaptive-encoder", "WebRTC encoder callback failed");
}
}
}
private:
std::mutex mutex_;
std::exception_ptr failure_;
};
class Readback final {
public:
Readback(ID3D11Device* device, Failure& failure) : device_(device), failure_(failure) {
device_->GetImmediateContext(&context_);
}
webrtc::scoped_refptr<webrtc::I420BufferInterface> Convert(ID3D11Texture2D* texture) {
try {
D3D11_TEXTURE2D_DESC description{};
texture->GetDesc(&description);
if (!staging_ || width_ != description.Width || height_ != description.Height) {
staging_.Reset();
description.Usage = D3D11_USAGE_STAGING;
description.BindFlags = 0;
description.CPUAccessFlags = D3D11_CPU_ACCESS_READ;
description.MiscFlags = 0;
Check(device_->CreateTexture2D(&description, nullptr, &staging_),
"adaptive-vp8-staging");
width_ = description.Width;
height_ = description.Height;
}
auto pixels = webrtc::I420Buffer::Create(static_cast<int>(width_),
static_cast<int>(height_));
context_->CopyResource(staging_.Get(), texture);
D3D11_MAPPED_SUBRESOURCE mapped{};
Check(context_->Map(staging_.Get(), 0, D3D11_MAP_READ, 0, &mapped),
"adaptive-vp8-readback");
const auto* y = static_cast<const UINT8*>(mapped.pData);
const int result = libyuv::NV12ToI420(
y, static_cast<int>(mapped.RowPitch), y + mapped.RowPitch * height_,
static_cast<int>(mapped.RowPitch), pixels->MutableDataY(), pixels->StrideY(),
pixels->MutableDataU(), pixels->StrideU(), pixels->MutableDataV(),
pixels->StrideV(), static_cast<int>(width_), static_cast<int>(height_));
context_->Unmap(staging_.Get(), 0);
if (result != 0) Fail("adaptive-vp8-convert", "NV12 conversion failed");
return pixels;
} catch (...) {
failure_.Capture();
return nullptr;
}
}
private:
ComPtr<ID3D11Device> device_;
ComPtr<ID3D11DeviceContext> context_;
ComPtr<ID3D11Texture2D> staging_;
Failure& failure_;
UINT32 width_ = 0, height_ = 0;
};
class NativeVideoFrameBuffer : public webrtc::VideoFrameBuffer {
public:
NativeVideoFrameBuffer(ComPtr<ID3D11Texture2D> texture, UINT32 width, UINT32 height,
UINT64 timestamp100ns, Readback& readback)
: texture_(std::move(texture)), width_(width), height_(height),
timestamp100ns_(timestamp100ns), readback_(readback) {}
Type type() const override { return Type::kNative; }
int width() const override { return static_cast<int>(width_); }
int height() const override { return static_cast<int>(height_); }
webrtc::scoped_refptr<webrtc::I420BufferInterface> ToI420() override {
return readback_.Convert(texture_.Get());
}
ID3D11Texture2D* texture() const { return texture_.Get(); }
UINT64 timestamp100ns() const { return timestamp100ns_; }
private:
ComPtr<ID3D11Texture2D> texture_;
const UINT32 width_, height_;
const UINT64 timestamp100ns_;
Readback& readback_;
};
class HardwareEncoder final : public webrtc::VideoEncoder {
public:
HardwareEncoder(const webrtc::Environment& env, VideoProfile ceiling,
AdaptiveEncoder::Factory create,
std::unique_ptr<screener::capture::windows::VideoEncoder> initial,
Failure& failure)
: env_(env), ceiling_(ceiling), create_(std::move(create)),
initial_(std::move(initial)), failure_(failure) {
Check(CoInitializeEx(nullptr, COINIT_MULTITHREADED), "adaptive-h264-apartment");
}
~HardwareEncoder() override {
initial_.reset();
Release();
CoUninitialize();
}
int32_t InitEncode(const webrtc::VideoCodec* config, const Settings&) override {
Release();
try {
if (!config || config->codecType != webrtc::kVideoCodecH264 ||
config->width == 0 || config->height == 0 || (config->width & 1) ||
(config->height & 1) || config->width > ceiling_.width ||
config->height > ceiling_.height || config->maxFramerate == 0 ||
config->maxFramerate > ceiling_.frame_rate || config->startBitrate == 0 ||
config->numberOfSimulcastStreams > 1 || config->H264().numberOfTemporalLayers > 1) {
Fail("adaptive-h264-config", "WebRTC requested an unsupported H264 configuration");
}
profile_ = ceiling_;
profile_.width = config->width;
profile_.height = config->height;
profile_.frame_rate = config->maxFramerate;
profile_.bit_rate = config->startBitrate * 1000;
if (initial_ && profile_.width == ceiling_.width &&
profile_.height == ceiling_.height && profile_.frame_rate == ceiling_.frame_rate) {
encoder_ = std::move(initial_);
encoder_->SetBitrate(profile_.bit_rate);
} else {
initial_.reset();
encoder_ = create_(profile_);
}
if (!encoder_ || encoder_->kind != OutputKind::h264)
Fail("adaptive-h264-create", "The native H264 encoder is unavailable");
applied_bitrate_ = profile_.bit_rate;
return WEBRTC_VIDEO_CODEC_OK;
} catch (...) {
failure_.Capture();
Release();
return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
}
}
int32_t RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback* callback) override {
callback_ = callback;
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t Release() override {
encoder_.reset();
paused_ = false;
key_frame_pending_ = true;
applied_bitrate_ = 0;
return WEBRTC_VIDEO_CODEC_OK;
}
void SetRates(const RateControlParameters& rates) override {
if (!encoder_) return;
try {
const UINT32 bitrate = rates.bitrate.get_sum_bps();
if (bitrate != rates.bitrate.GetBitrate(0, 0) ||
!std::isfinite(rates.framerate_fps))
Fail("adaptive-h264-rates", "WebRTC supplied unsupported H264 rates");
if (bitrate == 0) {
paused_ = true;
return;
}
const double fps = rates.framerate_fps > 0 ? rates.framerate_fps : profile_.frame_rate;
// Chromium M152 MF compensates bitrate when requested FPS differs from configured FPS.
const double adjusted = static_cast<double>(bitrate) * profile_.frame_rate / fps;
if (!std::isfinite(adjusted) || adjusted < 1 ||
adjusted > std::numeric_limits<UINT32>::max())
Fail("adaptive-h264-rates", "H264 rate compensation overflowed");
const auto next = static_cast<UINT32>(adjusted);
if (next != applied_bitrate_) encoder_->SetBitrate(next);
applied_bitrate_ = next;
key_frame_pending_ = key_frame_pending_ || paused_;
paused_ = false;
} catch (...) {
failure_.Capture();
}
}
int32_t Encode(const webrtc::VideoFrame& frame,
const std::vector<webrtc::VideoFrameType>* requested) override {
try {
failure_.Rethrow();
if (!encoder_ || !callback_)
Fail("adaptive-h264-encode", "The H264 encoder is not initialized");
key_frame_pending_ = key_frame_pending_ ||
(requested && std::find(requested->begin(), requested->end(),
webrtc::VideoFrameType::kVideoFrameKey) != requested->end());
if (paused_ || (requested && !requested->empty() &&
requested->front() == webrtc::VideoFrameType::kEmptyFrame)) {
callback_->OnFrameDropped(frame.rtp_timestamp(), 0, true);
return WEBRTC_VIDEO_CODEC_OK;
}
const auto* native = dynamic_cast<NativeVideoFrameBuffer*>(frame.video_frame_buffer().get());
if (!native || frame.width() != static_cast<int>(profile_.width) ||
frame.height() != static_cast<int>(profile_.height))
Fail("adaptive-h264-input", "H264 requires the selected native NV12 surface");
const auto began = env_.clock().TimeInMilliseconds();
const auto access_unit = encoder_->Encode(native->texture(), native->timestamp100ns(),
key_frame_pending_);
key_frame_pending_ = false;
webrtc::EncodedImage image;
image.SetEncodedData(webrtc::EncodedImageBuffer::Create(
access_unit.bytes.data(), access_unit.bytes.size()));
image._encodedWidth = profile_.width;
image._encodedHeight = profile_.height;
image.SetRtpTimestamp(frame.rtp_timestamp());
image.capture_time_ms_ = frame.timestamp_us() / 1000;
image.ntp_time_ms_ = frame.ntp_time_ms();
image.SetPresentationTimestamp(frame.presentation_timestamp());
image.rotation_ = frame.rotation();
image.SetColorSpace(frame.color_space());
image.SetVideoFrameTrackingId(frame.id());
image.SetSimulcastIndex(0);
image.SetSpatialIndex(0);
image.SetTemporalIndex(0);
image.set_end_of_temporal_unit(true);
image.SetFrameType(access_unit.key_frame ? webrtc::VideoFrameType::kVideoFrameKey
: webrtc::VideoFrameType::kVideoFrameDelta);
image.SetEncodeTime(began, env_.clock().TimeInMilliseconds());
webrtc::CodecSpecificInfo info;
info.codecType = webrtc::kVideoCodecH264;
info.codecSpecific.H264 = {};
info.codecSpecific.H264.packetization_mode = webrtc::H264PacketizationMode::NonInterleaved;
info.codecSpecific.H264.idr_frame = access_unit.key_frame;
info.scalability_mode = webrtc::ScalabilityMode::kL1T1;
if (callback_->OnEncodedImage(image, &info).error != webrtc::EncodedImageCallback::Result::OK)
Fail("adaptive-h264-callback", "WebRTC rejected the encoded H264 frame");
return WEBRTC_VIDEO_CODEC_OK;
} catch (...) {
failure_.Capture();
return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
}
}
EncoderInfo GetEncoderInfo() const override {
EncoderInfo info;
info.implementation_name = "Windows Media Foundation H264";
info.is_hardware_accelerated = true;
info.enable_cpu_overuse_detection = true;
info.has_trusted_rate_controller = false;
// VSE parses H264 slice QP; use upstream H264 quality-scaler thresholds.
info.is_qp_trusted = true;
info.scaling_settings = ScalingSettings(24, 37);
info.requested_resolution_alignment = 2;
info.supports_native_handle = true;
info.supports_simulcast = false;
info.preferred_pixel_formats = {webrtc::VideoFrameBuffer::Type::kNative};
return info;
}
private:
const webrtc::Environment env_;
const VideoProfile ceiling_;
AdaptiveEncoder::Factory create_;
std::unique_ptr<screener::capture::windows::VideoEncoder> initial_, encoder_;
Failure& failure_;
VideoProfile profile_;
webrtc::EncodedImageCallback* callback_ = nullptr;
UINT32 applied_bitrate_ = 0;
bool paused_ = false, key_frame_pending_ = true;
};
webrtc::DegradationPreference Preference(DegradationPreference preference) {
switch (preference) {
case DegradationPreference::resolution: return webrtc::DegradationPreference::MAINTAIN_RESOLUTION;
case DegradationPreference::framerate: return webrtc::DegradationPreference::MAINTAIN_FRAMERATE;
case DegradationPreference::balanced: return webrtc::DegradationPreference::BALANCED;
}
Fail("adaptive-preference", "Unknown degradation preference");
}
} // namespace
class AdaptiveEncoder::Impl final : public webrtc::VideoSourceInterface<webrtc::VideoFrame>,
public webrtc::VideoStreamEncoderInterface::EncoderSink,
public webrtc::VideoEncoderFactory {
public:
Impl(OutputKind kind, VideoProfile ceiling, ID3D11Device* device,
Factory create, std::unique_ptr<VideoEncoder> initial)
: kind_(kind), ceiling_(ceiling), device_(device), create_(std::move(create)),
initial_(std::move(initial)), env_(webrtc::CreateEnvironment()),
readback_(device, failure_), builtin_(webrtc::CreateBuiltinVideoEncoderFactory()),
allocator_(webrtc::CreateBuiltinVideoBitrateAllocatorFactory()),
worker_(env_.task_queue_factory().CreateTaskQueue(
"CaptureAdaptation", webrtc::TaskQueueFactory::Priority::kNormal)) {}
~Impl() override {
Sync(worker_.get(), [&] {
if (encoder_) {
encoder_->Stop();
encoder_.reset();
encode_queue_ = nullptr;
}
});
}
void Start() {
Sync(worker_.get(), [&] {
const auto format = GetSupportedFormats().front();
webrtc::VideoSendStream::Config send_config(nullptr);
send_config.rtp.ssrcs = {1};
send_config.rtp.payload_name = format.name;
send_config.rtp.payload_type = 96;
stats_ = std::make_unique<webrtc::SendStatisticsProxy>(
&env_.clock(), send_config, webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo,
env_.field_trials());
webrtc::VideoStreamEncoderSettings settings(webrtc::VideoEncoder::Capabilities(false));
settings.encoder_factory = this;
settings.bitrate_allocator_factory = allocator_.get();
auto queue = env_.task_queue_factory().CreateTaskQueue(
"CaptureEncoder", webrtc::TaskQueueFactory::Priority::kNormal);
encode_queue_ = queue.get();
auto cadence = webrtc::FrameCadenceAdapterInterface::Create(
&env_.clock(), queue.get(), nullptr, webrtc::TaskQueueBase::Current(), env_.field_trials());
encoder_ = std::make_unique<webrtc::VideoStreamEncoder>(
env_, static_cast<int>(std::max(1u, std::thread::hardware_concurrency())),
stats_.get(), settings, std::make_unique<webrtc::OveruseFrameDetector>(env_, stats_.get()),
std::move(cadence), std::move(queue),
webrtc::VideoStreamEncoder::BitrateAllocationCallbackType::kVideoBitrateAllocation,
nullptr, [this](std::optional<webrtc::SdpVideoFormat>, bool) {
failure_.Set("WebRTC could not continue with the selected encoder");
});
adapter_.OnOutputFormatRequest(std::nullopt,
static_cast<int>(ceiling_.width * ceiling_.height), static_cast<int>(ceiling_.frame_rate));
encoder_->SetSink(this, false);
encoder_->SetSource(this, Preference(ceiling_.preference));
encoder_->SetStartBitrate(static_cast<int>(ceiling_.bit_rate));
webrtc::VideoEncoderConfig config;
config.codec_type = kind_ == OutputKind::h264 ? webrtc::kVideoCodecH264 : webrtc::kVideoCodecVP8;
config.video_format = format;
config.content_type = webrtc::VideoEncoderConfig::ContentType::kRealtimeVideo;
config.number_of_streams = 1;
config.max_bitrate_bps = static_cast<int>(ceiling_.bit_rate);
config.frame_drop_enabled = true;
config.is_quality_scaling_allowed = true;
config.simulcast_layers.resize(1);
config.simulcast_layers[0].active = true;
config.simulcast_layers[0].max_framerate = static_cast<int>(ceiling_.frame_rate);
config.simulcast_layers[0].max_bitrate_bps = static_cast<int>(ceiling_.bit_rate);
config.simulcast_layers[0].num_temporal_layers = 1;
if (kind_ == OutputKind::vp8) {
auto vp8 = webrtc::VideoEncoder::GetDefaultVp8Settings();
vp8.automaticResizeOn = true;
config.encoder_specific_settings =
webrtc::make_ref_counted<webrtc::VideoEncoderConfig::Vp8EncoderSpecificSettings>(vp8);
}
encoder_->ConfigureEncoder(std::move(config), 1200, [this](webrtc::RTCError error) {
if (!error.ok()) failure_.Set("WebRTC encoder configuration failed");
});
});
Sync(encode_queue_, [] {});
failure_.Rethrow();
}
std::optional<AdaptiveAccessUnit> Encode(FrameProducer produce, UINT32 source_width,
UINT32 source_height, UINT64 timestamp100ns, bool key_frame, UINT32 bitrate) {
if (!produce || source_width == 0 || source_height == 0 ||
source_width > ceiling_.width || source_height > ceiling_.height ||
timestamp100ns > static_cast<UINT64>(std::numeric_limits<INT64>::max()))
Fail("adaptive-input", "Invalid adaptive encoder input");
failure_.Rethrow();
Sync(worker_.get(), [&] {
const auto rate = webrtc::DataRate::BitsPerSec(std::min(bitrate, ceiling_.bit_rate));
if (!bitrate_ || *bitrate_ != rate.bps()) {
encoder_->OnBitrateUpdated(rate, rate, 0, 0, 0);
bitrate_ = rate.bps();
}
if (key_frame) encoder_->SendKeyFrame();
if (!source_sink_) return;
if (!anchor100ns_) {
anchor100ns_ = timestamp100ns;
anchor_us_ = env_.clock().TimeInMicroseconds();
anchor_ntp_ms_ = env_.clock().CurrentNtpInMilliseconds();
}
if (timestamp100ns < *anchor100ns_)
Fail("adaptive-timestamp", "Capture timestamp moved before its output anchor");
const auto elapsed = timestamp100ns - *anchor100ns_;
const INT64 capture_us = anchor_us_ + static_cast<INT64>(elapsed / 10);
const INT64 ntp_ms = anchor_ntp_ms_ + static_cast<INT64>(elapsed / 10'000);
int crop_width, crop_height, width, height;
if (!adapter_.AdaptFrameResolution(static_cast<int>(source_width),
static_cast<int>(source_height), capture_us * 1000,
&crop_width, &crop_height, &width, &height)) {
source_sink_->OnDiscardedFrame();
return;
}
auto texture = produce(static_cast<UINT32>(width), static_cast<UINT32>(height));
if (!texture) Fail("adaptive-surface", "The selected NV12 surface is unavailable");
D3D11_TEXTURE2D_DESC description{};
texture->GetDesc(&description);
ComPtr<ID3D11Device> texture_device;
texture->GetDevice(&texture_device);
if (texture_device.Get() != device_.Get() || description.Format != DXGI_FORMAT_NV12 ||
description.Width != static_cast<UINT32>(width) ||
description.Height != static_cast<UINT32>(height))
Fail("adaptive-surface", "The selected NV12 surface has a different device or size");
{
std::lock_guard<std::mutex> lock(output_mutex_);
timestamp100ns_ = timestamp100ns;
rtp_timestamp_ = 90u * static_cast<UINT32>(ntp_ms);
const double fps = std::min<double>(ceiling_.frame_rate, adapter_.GetMaxFramerate());
duration100ns_ = static_cast<UINT64>(10'000'000 / fps);
}
auto buffer = webrtc::make_ref_counted<NativeVideoFrameBuffer>(
std::move(texture), static_cast<UINT32>(width), static_cast<UINT32>(height),
timestamp100ns, readback_);
source_sink_->OnFrame(webrtc::VideoFrame::Builder().set_video_frame_buffer(buffer)
.set_timestamp_us(capture_us).set_ntp_time_ms(ntp_ms).build());
});
// Both selected codecs complete synchronously; native frames cannot be retained by VSE.
Sync(encode_queue_, [] {});
Sync(worker_.get(), [] {});
failure_.Rethrow();
std::lock_guard<std::mutex> lock(output_mutex_);
return std::exchange(output_, std::nullopt);
}
void AddOrUpdateSink(webrtc::VideoSinkInterface<webrtc::VideoFrame>* sink,
const webrtc::VideoSinkWants& wants) override {
source_sink_ = sink;
adapter_.OnSinkWants(wants);
}
void RemoveSink(webrtc::VideoSinkInterface<webrtc::VideoFrame>* sink) override {
if (source_sink_ == sink) source_sink_ = nullptr;
}
std::vector<webrtc::SdpVideoFormat> GetSupportedFormats() const override {
if (kind_ == OutputKind::vp8) return {webrtc::SdpVideoFormat("VP8")};
return {webrtc::SdpVideoFormat("H264", {{"profile-level-id", ceiling_.profile_level_id()},
{"packetization-mode", "1"}, {"level-asymmetry-allowed", "1"}})};
}
std::unique_ptr<webrtc::VideoEncoder> Create(const webrtc::Environment& env,
const webrtc::SdpVideoFormat& format) override {
try {
if (kind_ == OutputKind::vp8) {
initial_.reset();
auto encoder = builtin_->Create(env, format);
if (!encoder) Fail("adaptive-vp8-create", "WebRTC VP8 encoder is unavailable");
return encoder;
}
return std::make_unique<HardwareEncoder>(env, ceiling_, create_, std::move(initial_), failure_);
} catch (...) {
failure_.Capture();
return nullptr;
}
}
void OnEncoderConfigurationChanged(std::vector<webrtc::VideoStream>, bool,
webrtc::VideoEncoderConfig::ContentType, int) override {}
void OnBitrateAllocationUpdated(const webrtc::VideoBitrateAllocation&) override {}
void OnVideoLayersAllocationUpdated(webrtc::VideoLayersAllocation) override {}
void OnFrameDropped(uint32_t, int, bool) override {}
Result OnEncodedImage(const webrtc::EncodedImage& image,
const webrtc::CodecSpecificInfo*) override {
try {
std::lock_guard<std::mutex> lock(output_mutex_);
if (output_)
Fail("adaptive-output-duplicate", "WebRTC returned more than one access unit for the current input");
if (image.RtpTimestamp() != rtp_timestamp_)
Fail("adaptive-output-timestamp", "WebRTC returned an access unit from a different input");
if (image.size() == 0 || image.size() > kMaxProductAccessUnitBytes)
Fail("adaptive-output-size", "Encoded access unit bytes=" + std::to_string(image.size()));
if (image._encodedWidth == 0 || image._encodedHeight == 0 ||
image._encodedWidth > ceiling_.width || image._encodedHeight > ceiling_.height)
Fail("adaptive-output-dimensions", "WebRTC returned output outside its configured dimensions");
AdaptiveAccessUnit output;
output.access_unit.timestamp100ns = timestamp100ns_;
output.access_unit.key_frame = image.IsKey();
output.access_unit.bytes.assign(image.data(), image.data() + image.size());
output.width = image._encodedWidth;
output.height = image._encodedHeight;
output.duration100ns = duration100ns_;
output_ = std::move(output);
return Result(Result::OK);
} catch (...) {
failure_.Capture();
return Result(Result::ERROR_SEND_FAILED);
}
}
private:
const OutputKind kind_;
const VideoProfile ceiling_;
ComPtr<ID3D11Device> device_;
Factory create_;
std::unique_ptr<VideoEncoder> initial_;
const webrtc::Environment env_;
Failure failure_;
Readback readback_;
std::unique_ptr<webrtc::VideoEncoderFactory> builtin_;
std::unique_ptr<webrtc::VideoBitrateAllocatorFactory> allocator_;
std::unique_ptr<webrtc::SendStatisticsProxy> stats_;
std::unique_ptr<webrtc::TaskQueueBase, webrtc::TaskQueueDeleter> worker_;
std::unique_ptr<webrtc::VideoStreamEncoder> encoder_;
webrtc::TaskQueueBase* encode_queue_ = nullptr;
webrtc::VideoSinkInterface<webrtc::VideoFrame>* source_sink_ = nullptr;
webrtc::VideoAdapter adapter_{2};
std::optional<INT64> bitrate_;
std::optional<UINT64> anchor100ns_;
INT64 anchor_us_ = 0, anchor_ntp_ms_ = 0;
std::mutex output_mutex_;
std::optional<AdaptiveAccessUnit> output_;
UINT64 timestamp100ns_ = 0, duration100ns_ = 0;
UINT32 rtp_timestamp_ = 0;
};
AdaptiveEncoder::AdaptiveEncoder(OutputKind kind, VideoProfile ceiling, ID3D11Device* device,
Factory create, std::unique_ptr<VideoEncoder> initial) {
if ((kind != OutputKind::vp8 && kind != OutputKind::h264) || !device ||
(kind == OutputKind::h264 && !create) ||
ceiling.width == 0 || ceiling.height == 0 || (ceiling.width & 1) || (ceiling.height & 1) ||
static_cast<UINT64>(ceiling.width) * ceiling.height > std::numeric_limits<int>::max() ||
ceiling.frame_rate == 0 || ceiling.frame_rate > 1'000 || ceiling.bit_rate < 1'000 ||
ceiling.bit_rate > static_cast<UINT32>(std::numeric_limits<int>::max()))
Fail("adaptive-profile", "Invalid adaptive encoder profile");
impl_ = std::make_unique<Impl>(kind, ceiling, device, std::move(create), std::move(initial));
impl_->Start();
}
AdaptiveEncoder::~AdaptiveEncoder() = default;
std::optional<AdaptiveAccessUnit> AdaptiveEncoder::Encode(FrameProducer produce,
UINT32 source_width, UINT32 source_height, UINT64 timestamp100ns,
bool key_frame, UINT32 bitrate) {
return impl_->Encode(std::move(produce), source_width, source_height, timestamp100ns,
key_frame, bitrate);
}
} // namespace screener::capture::windows
native/capture/windows/adaptive_encoder.h
+38
View File
@@ -0,0 +1,38 @@
#pragma once
#include "h264_encoder.h"
#include <functional>
#include <memory>
namespace screener::capture::windows {
struct AdaptiveAccessUnit final {
EncodedAccessUnit access_unit;
UINT32 width = 0;
UINT32 height = 0;
UINT64 duration100ns = 0;
};
// One complete WebRTC encoding/adaptation pipeline for one shared output.
class AdaptiveEncoder final {
public:
using Factory = std::function<std::unique_ptr<VideoEncoder>(const VideoProfile&)>;
using FrameProducer = std::function<ComPtr<ID3D11Texture2D>(UINT32, UINT32)>;
AdaptiveEncoder(OutputKind kind, VideoProfile ceiling, ID3D11Device* device,
Factory create, std::unique_ptr<VideoEncoder> initial = nullptr);
~AdaptiveEncoder();
AdaptiveEncoder(const AdaptiveEncoder&) = delete;
AdaptiveEncoder& operator=(const AdaptiveEncoder&) = delete;
std::optional<AdaptiveAccessUnit> Encode(
FrameProducer produce, UINT32 source_width, UINT32 source_height,
UINT64 timestamp100ns, bool key_frame, UINT32 bitrate);
private:
class Impl;
std::unique_ptr<Impl> impl_;
};
} // namespace screener::capture::windows
native/capture/windows/build-libvpx.ps1
-68
View File
@@ -1,68 +0,0 @@
[CmdletBinding()]
param(
[Parameter(Mandatory = $true)] [string]$OutputDirectory,
[Parameter(Mandatory = $true)] [string]$VisualStudioDirectory
)
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
$ProgressPreference = 'SilentlyContinue'
$pinPath = Join-Path $PSScriptRoot 'libvpx-dependencies.json'
$pins = Get-Content -Raw $pinPath | ConvertFrom-Json
$outputPath = [IO.Path]::GetFullPath($OutputDirectory)
$sourcePath = Join-Path $outputPath ('libvpx-' + $pins.libvpx.commit)
$buildPath = Join-Path $outputPath 'build'
$libraryPath = Join-Path $buildPath 'x64\Release\vpxmt.lib'
$stampPath = Join-Path $buildPath 'build-stamp.txt'
$toolset = Get-ChildItem (Join-Path $VisualStudioDirectory 'VC\Tools\MSVC') -Directory |
Sort-Object Name -Descending | Select-Object -First 1
$stamp = (Get-FileHash $pinPath).Hash + (Get-FileHash $PSCommandPath).Hash + $toolset.Name
if ((Test-Path $libraryPath) -and (Test-Path $stampPath) -and
(Get-Content -Raw $stampPath).Trim() -eq $stamp) {
return @{ Include = $sourcePath; Library = $libraryPath }
}
New-Item -ItemType Directory -Path $outputPath,$buildPath -Force | Out-Null
function Get-PinnedArchive($pin, [string]$name) {
$archive = Join-Path $outputPath $name
if (!(Test-Path $archive) -or
(Get-FileHash $archive -Algorithm SHA256).Hash.ToLowerInvariant() -ne $pin.sha256) {
Invoke-WebRequest -UseBasicParsing -Uri $pin.url -OutFile $archive -TimeoutSec 120
}
if ((Get-FileHash $archive -Algorithm SHA256).Hash.ToLowerInvariant() -ne $pin.sha256) {
throw "Native codec dependency digest mismatch: $name"
}
return $archive
}
$sourceArchive = Get-PinnedArchive $pins.libvpx 'libvpx.tar.gz'
& tar -xf $sourceArchive -C $outputPath
if ($LASTEXITCODE -ne 0) { throw 'Could not unpack libvpx source.' }
$nasmArchive = Get-PinnedArchive $pins.nasm 'nasm.zip'
$nasmPath = Join-Path $outputPath ('nasm\nasm-' + $pins.nasm.version)
Expand-Archive -LiteralPath $nasmArchive -DestinationPath (Join-Path $outputPath 'nasm') -Force
$makeArchive = Get-PinnedArchive $pins.make 'make.pkg.tar.zst'
$makeRoot = Join-Path $outputPath 'make'
$makePath = Join-Path $makeRoot 'usr\bin'
New-Item -ItemType Directory -Path $makeRoot -Force | Out-Null
& tar -xf $makeArchive -C $makeRoot usr/bin/make.exe
if ($LASTEXITCODE -ne 0) { throw 'Could not unpack the libvpx build tool.' }
$gitRoot = Split-Path -Parent (Split-Path -Parent (Get-Command git.exe).Source)
$bashPath = Join-Path $gitRoot 'bin\bash.exe'
if (!(Test-Path $bashPath)) { throw 'Git for Windows Bash is required to build libvpx.' }
$developerCommand = Join-Path $VisualStudioDirectory 'Common7\Tools\VsDevCmd.bat'
$compile = @(
'set "PATH={0};{1};{2};%PATH%"' -f $nasmPath,$makePath,(Join-Path $gitRoot 'usr\bin')
'call "{0}" -arch=x64 -host_arch=x64 >nul' -f $developerCommand
'cd /d "{0}"' -f $buildPath
'"{0}" "{1}" --target=x86_64-win64-vs17 --as=nasm --disable-vp9 --disable-vp8-decoder --disable-examples --disable-unit-tests --disable-tools --disable-docs --disable-webm-io --disable-libyuv --disable-postproc --enable-realtime-only --enable-static-msvcrt' -f $bashPath,((Join-Path $sourcePath 'configure').Replace('\','/'))
'make.exe -B vpx.vcxproj'
'msbuild vpx.vcxproj /nologo /t:Rebuild /p:Configuration=Release /p:Platform=x64 /m:4 /v:minimal'
) -join ' && '
& cmd.exe /d /s /c $compile | Out-Host
if ($LASTEXITCODE -ne 0 -or !(Test-Path $libraryPath)) {
throw 'Native VP8 library compilation failed.'
}
[IO.File]::WriteAllText($stampPath, $stamp + "`n")
return @{ Include = $sourcePath; Library = $libraryPath }
native/capture/windows/build.ps1
+41 -33
View File
@@ -9,12 +9,14 @@ Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
$helperDirectory = Split-Path -Parent $MyInvocation.MyCommand.Path
$repositoryRoot = [IO.Path]::GetFullPath((Join-Path $helperDirectory '..\..'))
$repositoryRoot = [IO.Path]::GetFullPath((Join-Path $helperDirectory '..\..\..'))
$outputPath = [IO.Path]::GetFullPath($OutputDirectory)
$repositoryPrefix = $repositoryRoot.TrimEnd([IO.Path]::DirectorySeparatorChar) + [IO.Path]::DirectorySeparatorChar
$buildPrefix = (Join-Path $repositoryRoot 'build') + [IO.Path]::DirectorySeparatorChar
if ($outputPath.Equals($repositoryRoot, [StringComparison]::OrdinalIgnoreCase) -or
$outputPath.StartsWith($repositoryPrefix, [StringComparison]::OrdinalIgnoreCase)) {
throw 'OutputDirectory must be outside the repository.'
($outputPath.StartsWith($repositoryPrefix, [StringComparison]::OrdinalIgnoreCase) -and
!$outputPath.StartsWith($buildPrefix, [StringComparison]::OrdinalIgnoreCase))) {
throw 'OutputDirectory must be below build/ or outside the repository.'
}
$vswhere = Join-Path ${env:ProgramFiles(x86)} 'Microsoft Visual Studio\Installer\vswhere.exe'
@@ -24,39 +26,45 @@ if ([string]::IsNullOrWhiteSpace($installationPath)) {
}
$developerCommand = Join-Path $installationPath 'Common7\Tools\VsDevCmd.bat'
$mainSourcePath = Join-Path $helperDirectory 'main.cpp'
$audioSourcePath = Join-Path $helperDirectory 'process_audio.cpp'
$targetSourcePath = Join-Path $helperDirectory 'capture_target.cpp'
$vp8SourcePath = Join-Path $helperDirectory 'vp8_encoder.cpp'
$executablePath = Join-Path $outputPath 'screener-client-capture.exe'
$mainObjectPath = Join-Path $outputPath 'window-capture.obj'
$audioObjectPath = Join-Path $outputPath 'process-audio.obj'
$targetObjectPath = Join-Path $outputPath 'capture-target.obj'
$vp8ObjectPath = Join-Path $outputPath 'vp8-encoder.obj'
New-Item -ItemType Directory -Path $outputPath -Force | Out-Null
$vpx = & (Join-Path $helperDirectory 'build-libvpx.ps1') -OutputDirectory (Join-Path $outputPath 'libvpx') -VisualStudioDirectory $installationPath
$compile = @(
'call "{0}" -arch=x64 -host_arch=x64 >nul' -f $developerCommand
'cl.exe /nologo /c /std:c++20 /EHsc /W4 /WX /external:I "{2}" /external:W0 /DUNICODE /D_UNICODE /DWIN32_LEAN_AND_MEAN /D_WIN32_WINNT=0x0A00 /DNTDDI_VERSION=0x0A00000A "{0}" /Fo:"{1}"' -f $mainSourcePath, $mainObjectPath, $vpx.Include
'cl.exe /nologo /c /std:c++20 /EHsc /W4 /WX /DUNICODE /D_UNICODE /DWIN32_LEAN_AND_MEAN /D_WIN32_WINNT=0x0A00 /DNTDDI_VERSION=0x0A00000A "{0}" /Fo:"{1}"' -f $audioSourcePath, $audioObjectPath
'cl.exe /nologo /c /std:c++20 /EHsc /W4 /WX /DUNICODE /D_UNICODE /DWIN32_LEAN_AND_MEAN /D_WIN32_WINNT=0x0A00 /DNTDDI_VERSION=0x0A00000A "{0}" /Fo:"{1}"' -f $targetSourcePath, $targetObjectPath
'cl.exe /nologo /c /std:c++20 /EHsc /O2 /W4 /WX /external:I "{2}" /external:W0 "{0}" /Fo:"{1}"' -f $vp8SourcePath, $vp8ObjectPath, $vpx.Include
'link.exe /nologo /LTCG /out:"{0}" "{1}" "{2}" "{3}" "{4}" "{5}" ole32.lib mmdevapi.lib runtimeobject.lib user32.lib gdi32.lib dwmapi.lib shell32.lib mfplat.lib mf.lib mfuuid.lib d3d11.lib dxgi.lib dxguid.lib evr.lib oleaut32.lib windowsapp.lib' -f $executablePath, $mainObjectPath, $audioObjectPath, $targetObjectPath, $vp8ObjectPath, $vpx.Library
) -join ' && '
& cmd.exe /d /s /c $compile
if ($LASTEXITCODE -ne 0) {
throw "Window-capture helper compilation failed with exit code $LASTEXITCODE."
$webrtc = & (Join-Path $helperDirectory 'get-webrtc.ps1')
$includeFlags = @('', 'third_party\abseil-cpp', 'third_party\boringssl\src\include',
'third_party\libyuv\include', 'third_party\libvpx\source\libvpx') | ForEach-Object {
'/external:I "{0}"' -f (Join-Path $webrtc.Include $_).TrimEnd('\')
}
$compileFlags = '/nologo /c /std:c++20 /EHsc /GR /O2 /W4 /WX /MT /D_ITERATOR_DEBUG_LEVEL=0 /DUNICODE /D_UNICODE /DWIN32_LEAN_AND_MEAN /D_WIN32_WINNT=0x0A00 /DNTDDI_VERSION=0x0A00000A /external:W0 ' + ($includeFlags -join ' ')
$systemLibraries = 'ole32.lib mmdevapi.lib runtimeobject.lib user32.lib gdi32.lib dwmapi.lib shell32.lib mfplat.lib mf.lib mfuuid.lib d3d11.lib dxgi.lib dxguid.lib evr.lib oleaut32.lib windowsapp.lib winmm.lib ws2_32.lib strmiids.lib crypt32.lib dmoguids.lib iphlpapi.lib msdmo.lib secur32.lib wmcodecdspuuid.lib'
$linkCommand = '"{0}" /nologo /libpath:"{1}"' -f $webrtc.Linker,$webrtc.RuntimeLibraries
function Invoke-CaptureBuild([string]$command) {
& cmd.exe /d /s /c ('call "{0}" -arch=x64 -host_arch=x64 >nul && {1}' -f $developerCommand,$command) | Out-Host
if ($LASTEXITCODE -ne 0) {
throw "Window-capture helper compilation failed with exit code $LASTEXITCODE."
}
}
$objects = foreach ($name in @('main', 'process_audio', 'capture_target', 'h264_encoder', 'adaptive_encoder')) {
$source = Join-Path $helperDirectory ($name + '.cpp')
$object = Join-Path $outputPath ($name + '.obj')
$definitions = if ($name -eq 'adaptive_encoder') { '/DWEBRTC_WIN /DRTC_ENABLE_H265 /DNOMINMAX' } else { '' }
Invoke-CaptureBuild ('cl.exe {0} /DNDEBUG {1} "{2}" /Fo:"{3}"' -f $compileFlags,$definitions,$source,$object)
'"{0}"' -f $object
}
Invoke-CaptureBuild ('{0} /out:"{1}" {2} "{3}" {4}' -f $linkCommand,$executablePath,($objects -join ' '),$webrtc.Library,$systemLibraries)
if ($Check) {
$geometrySource = Join-Path $helperDirectory 'capture_geometry.test.cpp'
$geometryObject = Join-Path $outputPath 'capture-geometry.test.obj'
$geometryCheck = Join-Path $outputPath 'capture-geometry.test.exe'
$compileCheck = 'call "{0}" -arch=x64 -host_arch=x64 >nul && cl.exe /nologo /std:c++20 /EHsc /W4 /WX "{1}" /Fo:"{2}" /Fe:"{3}" /link user32.lib' -f $developerCommand, $geometrySource, $geometryObject, $geometryCheck
& cmd.exe /d /s /c $compileCheck
if ($LASTEXITCODE -ne 0) { throw 'Capture geometry check compilation failed.' }
& $geometryCheck
if ($LASTEXITCODE -ne 0) { throw 'Capture geometry check failed.' }
$vp8Object = Join-Path $outputPath 'vp8_encoder.obj'
Invoke-CaptureBuild ('cl.exe {0} /DNDEBUG "{1}" /Fo:"{2}"' -f $compileFlags,(Join-Path $helperDirectory 'vp8_encoder.cpp'),$vp8Object)
foreach ($name in @('capture_geometry', 'capture_control')) {
$source = Join-Path $helperDirectory ($name + '.test.cpp')
$object = Join-Path $outputPath ($name + '.test.obj')
$executable = Join-Path $outputPath ($name + '.test.exe')
Invoke-CaptureBuild ('cl.exe {0} "{1}" /Fo:"{2}"' -f $compileFlags,$source,$object)
$codecObjects = if ($name -eq 'capture_control') { '"{0}" "{1}"' -f $vp8Object,$webrtc.Library } else { '' }
Invoke-CaptureBuild ('{0} /out:"{1}" "{2}" {3} {4}' -f $linkCommand,$executable,$object,$codecObjects,$systemLibraries)
& $executable
if ($LASTEXITCODE -ne 0) { throw "Capture check failed: $name" }
}
}
Write-Output $executablePath
native/capture/windows/capture_control.h
+58
View File
@@ -0,0 +1,58 @@
#pragma once
#include <cstdint>
#include <sstream>
#include <stdexcept>
#include <string>
#include <string_view>
namespace screener::capture {
constexpr int kMaxOutputs = 6;
struct CaptureControl final {
char kind = 'Q';
int layer = -1;
uint32_t value = 0;
};
// Control payload grammar inside one SMED input envelope.
class CaptureControls final {
public:
static CaptureControl Parse(std::string_view text) {
if (text.empty() || text.size() > 64) throw std::runtime_error("Invalid capture control size");
for (unsigned char byte : text) {
if (byte < 32 || byte > 126) throw std::runtime_error("Invalid capture control text");
}
std::istringstream input{std::string(text)};
CaptureControl control;
std::string kind;
int64_t value = 0;
bool valid = static_cast<bool>(input >> kind) && kind.size() == 1;
control.kind = kind.empty() ? 0 : kind.front();
switch (control.kind) {
case 'Q': break;
case 'K':
valid = valid && static_cast<bool>(input >> control.layer) &&
control.layer >= -1 && control.layer < kMaxOutputs;
break;
case 'A':
valid = valid && static_cast<bool>(input >> control.layer >> value) &&
control.layer >= 0 && control.layer < kMaxOutputs && value >= 0 && value <= 1;
control.value = static_cast<uint32_t>(value);
break;
case 'B':
valid = valid && static_cast<bool>(input >> control.layer >> value) &&
control.layer >= 0 && control.layer < kMaxOutputs && value >= 1'000 &&
value <= UINT32_MAX;
control.value = static_cast<uint32_t>(value);
break;
default: valid = false;
}
input >> std::ws;
if (!valid || !input.eof()) throw std::runtime_error("Invalid capture control command");
return control;
}
};
} // namespace screener::capture
native/capture/windows/capture_control.test.cpp
+244
View File
@@ -0,0 +1,244 @@
#include "capture_control.h"
#include "capture_input.h"
#include "capture_output.h"
#include "output_mailbox.h"
#include "vp8_encoder.h"
#include "vp8_decoder.h"
#include <array>
#include <cassert>
#include <chrono>
#include <future>
#include <thread>
#include <vector>
void CheckOutputPipe() {
HANDLE input = nullptr, output = nullptr;
assert(CreatePipe(&input, &output, nullptr, 4096));
screener::capture::ProtocolWriter writer(output);
assert(SUCCEEDED(writer.WriteBegin(333'333, 333'333)));
const std::array<BYTE, 3> payload{1, 2, 3};
assert(SUCCEEDED(writer.Write(screener::capture::OutputKind::vp8, 1,
333'333, 333'333, payload.data(), static_cast<DWORD>(payload.size()), 1, 320, 180)));
std::array<BYTE, 67> data{};
DWORD received = 0;
assert(ReadFile(input, data.data(), static_cast<DWORD>(data.size()), &received, nullptr));
assert(received == data.size() && data[4] == 2 && data[5] == 5 && data[31] == 0);
assert(data[37] == 4 && data[38] == 1 && data[39] == 1);
assert(data[56] == 1 && data[57] == 64 && data[58] == 0 && data[59] == 180 && data[63] == 3);
std::vector<BYTE> large(256 * 1024, 128);
HRESULT result = S_OK;
std::thread blocked([&]() {
result = writer.Write(screener::capture::OutputKind::vp8, 0,
666'666, 333'333, large.data(), static_cast<DWORD>(large.size()), 1, 320, 180);
});
const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(2);
DWORD available = 0;
while (available <= 32) {
assert(std::chrono::steady_clock::now() < deadline);
assert(PeekNamedPipe(input, nullptr, 0, nullptr, &available, nullptr));
std::this_thread::yield();
}
assert(WaitForSingleObject(blocked.native_handle(), 0) == WAIT_TIMEOUT);
writer.Stop();
do {
CancelSynchronousIo(blocked.native_handle());
assert(std::chrono::steady_clock::now() < deadline);
} while (WaitForSingleObject(blocked.native_handle(), 10) == WAIT_TIMEOUT);
blocked.join();
assert(FAILED(result));
CloseHandle(input);
CloseHandle(output);
}
void CheckWorkers() {
struct Input { uint32_t index; };
using Mailbox = screener::capture::OutputMailbox<Input>;
std::array<Mailbox, 3> mailboxes{Mailbox(90'000), Mailbox(300'000), Mailbox(1'200'000)};
std::array<std::thread, 3> workers;
std::array<uint32_t, 3> last{}, count{}, retired{};
std::array<bool, 3> recovery{};
std::mutex mutex;
std::condition_variable changed;
std::exception_ptr failure;
std::promise<void> entered, release;
auto entered_future = entered.get_future();
auto released = release.get_future().share();
for (size_t layer = 0; layer < workers.size(); ++layer) {
workers[layer] = std::thread([&, layer]() {
try {
const uint32_t width = 160u << layer;
const uint32_t height = 90u << layer;
std::vector<uint8_t> pixels(static_cast<size_t>(width) * height * 3 / 2, 128);
std::unique_ptr<screener::capture::Vp8Encoder> encoder;
for (;;) {
const auto work = mailboxes[layer].Take();
if (work.action == Mailbox::Action::stop) break;
if (work.action == Mailbox::Action::retire) {
encoder.reset();
std::lock_guard<std::mutex> lock(mutex);
++retired[layer];
changed.notify_one();
continue;
}
if (layer == 0 && work.input->index == 1) {
entered.set_value();
released.wait();
}
if (!encoder) encoder = std::make_unique<screener::capture::Vp8Encoder>(width, height, 30, work.bitrate);
encoder->SetBitrate(work.bitrate);
const auto output = encoder->Encode(pixels.data(), width,
static_cast<uint64_t>(work.input->index) * 10'000'000 / 30, work.recovery);
if (!mailboxes[layer].Accept(work.generation)) continue;
std::lock_guard<std::mutex> lock(mutex);
last[layer] = work.input->index;
recovery[layer] = output.key_frame;
++count[layer];
changed.notify_one();
}
} catch (...) {
std::lock_guard<std::mutex> lock(mutex);
failure = std::current_exception();
changed.notify_one();
}
});
}
auto submit = [&](uint32_t index) {
auto input = std::make_shared<Input>(Input{index});
for (auto& mailbox : mailboxes) mailbox.Submit(input);
};
auto wait = [&](auto predicate) {
std::unique_lock<std::mutex> lock(mutex);
const bool ready = changed.wait_for(lock, std::chrono::seconds(2), [&]() { return failure || predicate(); });
assert(ready && !failure);
};
submit(1);
assert(entered_future.wait_for(std::chrono::seconds(2)) == std::future_status::ready);
for (uint32_t index = 2; index <= 4; ++index) submit(index);
wait([&]() { return last[1] == 4 && last[2] == 4; });
mailboxes[0].RequestKeyFrame();
release.set_value();
wait([&]() { return last[0] == 4; });
assert(count[0] == 2 && recovery[0]); // New request survived the older keyframe; pending 2/3 were replaced.
mailboxes[1].SetActive(false);
mailboxes[2].SetActive(false);
submit(5);
wait([&]() { return last[0] == 5 && retired[1] == 1 && retired[2] == 1; });
assert(last[1] == 4 && last[2] == 4);
mailboxes[0].SetBitrate(30'000);
for (auto& mailbox : mailboxes) { mailbox.SetActive(true); mailbox.RequestKeyFrame(); }
submit(6);
wait([&]() { return last[0] == 6 && last[1] == 6 && last[2] == 6; });
assert(recovery[0] && recovery[1] && recovery[2]);
const auto stopped = std::chrono::steady_clock::now();
for (auto& mailbox : mailboxes) mailbox.Stop();
for (auto& worker : workers) worker.join();
assert(std::chrono::steady_clock::now() - stopped < std::chrono::seconds(2));
}
void CheckDroppedRecoveryInput() {
using Mailbox = screener::capture::OutputMailbox<int>;
Mailbox mailbox(90'000);
mailbox.Submit(std::make_shared<int>(1));
const auto dropped = mailbox.Take();
assert(dropped.recovery && *dropped.input == 1);
// The codec accepted input but produced no AU; preserve its consumed request.
if (dropped.recovery) mailbox.RequestKeyFrame();
mailbox.Submit(std::make_shared<int>(2));
const auto recovery = mailbox.Take();
assert(recovery.recovery && *recovery.input == 2);
assert(mailbox.Accept(recovery.generation));
mailbox.Submit(std::make_shared<int>(3));
assert(!mailbox.Take().recovery);
mailbox.Stop();
}
void CheckIndependentActivation() {
using Mailbox = screener::capture::OutputMailbox<int>;
Mailbox original(300'000), extra(90'000, false);
original.Submit(std::make_shared<int>(1));
const auto original_frame = original.Take();
extra.Submit(std::make_shared<int>(1));
auto pending = std::async(std::launch::async, [&]() { return extra.Take(); });
assert(pending.wait_for(std::chrono::milliseconds(20)) == std::future_status::timeout);
assert(extra.SetActive(true));
assert(!extra.SetActive(true));
extra.SetBitrate(30'000);
extra.Submit(std::make_shared<int>(2));
assert(pending.wait_for(std::chrono::seconds(2)) == std::future_status::ready);
const auto activated = pending.get();
assert(activated.action == Mailbox::Action::frame && *activated.input == 2 &&
activated.recovery && activated.bitrate == 30'000);
assert(!extra.SetActive(false));
assert(!extra.Accept(activated.generation) && original.Accept(original_frame.generation));
assert(extra.Take().action == Mailbox::Action::retire);
assert(extra.SetActive(true));
extra.Submit(std::make_shared<int>(3));
const auto replacement = extra.Take();
assert(replacement.recovery && *replacement.input == 3 &&
replacement.generation != activated.generation);
original.Stop();
extra.Stop();
}
void CheckInputEnvelopes() {
std::string control(32, '\0');
control.replace(0, 4, "SMED");
control[4] = 2;
control[5] = 7;
control[31] = 4;
control += "K -1";
screener::capture::InputEnvelopes parser;
assert(parser.Feed(std::string_view(control).substr(0, 11)).empty());
assert(!parser.Empty());
const auto controls = parser.Feed(std::string_view(control).substr(11));
assert(controls.size() == 1 && controls[0].Control().kind == 'K' && controls[0].Control().layer == -1);
assert(parser.Empty());
auto invalid = control;
invalid[7] = 1;
bool rejected = false;
try { (void)parser.Feed(invalid); } catch (const std::runtime_error&) { rejected = true; }
assert(rejected);
}
int main() {
using screener::capture::CaptureControls;
assert(CaptureControls::Parse("K -1").layer == -1);
assert(CaptureControls::Parse("A 5 1").layer == 5);
assert(CaptureControls::Parse("A 5 1").value == 1);
assert(CaptureControls::Parse("A 0 0").value == 0);
assert(CaptureControls::Parse("K 5").layer == 5);
assert(CaptureControls::Parse("B 1 90000").value == 90'000);
assert(CaptureControls::Parse("Q").kind == 'Q');
for (const auto& bad : {"K 6", "A 2", "A 6 1", "A -1 1", "A 0 2", "A2", "B 6 1000", "B 0 -1", "Q ignored", "\n"}) {
bool rejected = false;
try { (void)CaptureControls::Parse(bad); }
catch (const std::runtime_error&) { rejected = true; }
assert(rejected);
}
bool rejected = false;
try { (void)CaptureControls::Parse(std::string(65, 'K')); }
catch (const std::runtime_error&) { rejected = true; }
assert(rejected);
screener::capture::Vp8Encoder encoder(160, 90, 30, 90'000);
screener::capture::Vp8Decoder decoder;
std::vector<uint8_t> pixels(160 * 90 * 3 / 2, 128);
for (uint32_t frame = 0; frame < 12; ++frame) {
if (frame == 4) encoder.SetBitrate(30'000);
if (frame == 8) encoder.SetBitrate(90'000);
pixels[frame] = static_cast<uint8_t>(frame);
const bool recovery = frame % 4 == 0;
const auto output = encoder.Encode(pixels.data(), 160,
(static_cast<uint64_t>(frame) + 1) * 10'000'000 / 30, recovery);
assert(!output.bytes.empty() && (!recovery || output.key_frame));
const auto decoded = decoder.Decode(output.bytes);
assert(decoded.width == 160 && decoded.height == 90 && decoded.nv12.size() == pixels.size());
}
CheckWorkers();
CheckDroppedRecoveryInput();
CheckIndependentActivation();
CheckOutputPipe();
CheckInputEnvelopes();
}
native/capture/windows/capture_input.h
+101
View File
@@ -0,0 +1,101 @@
#pragma once
#include "capture_control.h"
#include <algorithm>
#include <array>
#include <cstdint>
#include <stdexcept>
#include <string_view>
#include <utility>
#include <vector>
namespace screener::capture {
struct InputEnvelope final {
uint8_t kind = 0;
bool recovery = false;
uint64_t timestamp = 0;
uint64_t duration = 0;
uint16_t width = 0;
uint16_t height = 0;
std::vector<uint8_t> data;
CaptureControl Control() const {
if (kind != 7) throw std::runtime_error("Expected a capture control envelope");
return CaptureControls::Parse(std::string_view(
reinterpret_cast<const char*>(data.data()), data.size()));
}
};
// The private stdin contract has one framing format for controls and encoded AUs.
// Retain at most one incomplete, bounded envelope between pipe reads.
class InputEnvelopes final {
public:
bool Empty() const { return header_size_ == 0 && frame_.data.empty(); }
std::vector<InputEnvelope> Feed(std::string_view bytes) {
std::vector<InputEnvelope> frames;
while (!bytes.empty()) {
if (header_size_ < header_.size()) {
const size_t take = std::min(header_.size() - header_size_, bytes.size());
std::copy_n(bytes.data(), take, header_.data() + header_size_);
header_size_ += take;
bytes.remove_prefix(take);
if (header_size_ != header_.size()) break;
ReadHeader();
}
const size_t take = std::min(payload_size_ - frame_.data.size(), bytes.size());
frame_.data.insert(frame_.data.end(), bytes.data(), bytes.data() + take);
bytes.remove_prefix(take);
if (frame_.data.size() == payload_size_) {
if (frame_.kind == 7) (void)frame_.Control();
frames.push_back(std::move(frame_));
frame_ = {};
header_size_ = 0;
}
}
return frames;
}
private:
uint64_t ReadBE(size_t offset, size_t count) const {
uint64_t result = 0;
for (size_t index = 0; index < count; ++index) result = result << 8 | header_[offset + index];
return result;
}
void ReadHeader() {
frame_.kind = header_[5];
frame_.recovery = header_[6] == 1;
frame_.timestamp = ReadBE(8, 8);
frame_.duration = ReadBE(16, 8);
frame_.width = static_cast<uint16_t>(ReadBE(24, 2));
frame_.height = static_cast<uint16_t>(ReadBE(26, 2));
payload_size_ = static_cast<size_t>(ReadBE(28, 4));
if (header_[0] != 'S' || header_[1] != 'M' || header_[2] != 'E' || header_[3] != 'D' ||
header_[4] != 2 || header_[7] != 0 || payload_size_ == 0) {
throw std::runtime_error("Invalid native input envelope");
}
if (frame_.kind == 7) {
if (header_[6] != 0 || frame_.timestamp != 0 || frame_.duration != 0 ||
frame_.width != 0 || frame_.height != 0 || payload_size_ > 64) {
throw std::runtime_error("Invalid native control envelope");
}
} else if ((frame_.kind != 2 && frame_.kind != 4) || header_[6] > 1 ||
frame_.duration == 0 || frame_.timestamp > static_cast<uint64_t>(INT64_MAX) / 100 ||
frame_.duration > static_cast<uint64_t>(INT64_MAX) / 100 ||
frame_.width < 2 || frame_.height < 2 || frame_.width > 2560 || frame_.height > 1440 ||
(frame_.width & 1) || (frame_.height & 1) || payload_size_ > 4 * 1024 * 1024) {
throw std::runtime_error("Invalid native encoded input");
}
frame_.data.reserve(payload_size_);
}
std::array<uint8_t, 32> header_{};
size_t header_size_ = 0;
size_t payload_size_ = 0;
InputEnvelope frame_;
};
} // namespace screener::capture
native/capture/windows/capture_output.h
+88
View File
@@ -0,0 +1,88 @@
#pragma once
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#include <algorithm>
#include <array>
#include <atomic>
#include <mutex>
#include <string>
namespace screener::capture {
constexpr DWORD kMaxProductAccessUnitBytes = 4 * 1024 * 1024;
constexpr DWORD kMaxStatusBytes = 4 * 1024;
enum class OutputKind : UINT8 { pcm = 1, h264 = 2, status = 3, vp8 = 4, begin = 5, unavailable = 6 };
// The caller owns the handle. One lock keeps each SMED envelope contiguous.
class ProtocolWriter final {
public:
explicit ProtocolWriter(HANDLE output = GetStdHandle(STD_OUTPUT_HANDLE)) : output_(output) {}
void Stop() { stopped_.store(true); }
HRESULT Write(OutputKind kind, UINT8 flags, UINT64 timestamp100ns,
UINT64 duration100ns, const BYTE* data, DWORD size,
UINT8 layer = 0, UINT16 width = 0, UINT16 height = 0) {
const bool diagnostic = kind == OutputKind::status || kind == OutputKind::unavailable;
const DWORD maximum = diagnostic ? kMaxStatusBytes : kMaxProductAccessUnitBytes;
if (size > maximum || (kind == OutputKind::begin ? size != 0 : data == nullptr || size == 0) ||
(!diagnostic && duration100ns == 0)) return HRESULT_FROM_WIN32(ERROR_INVALID_DATA);
std::array<BYTE, 32> header{'S', 'M', 'E', 'D', 2, static_cast<BYTE>(kind), flags, layer};
PutBE(header.data() + 8, timestamp100ns, 8);
PutBE(header.data() + 16, duration100ns, 8);
PutBE(header.data() + 24, width, 2);
PutBE(header.data() + 26, height, 2);
PutBE(header.data() + 28, size, 4);
std::lock_guard<std::mutex> lock(mutex_);
const HRESULT result = WriteAll(header.data(), static_cast<DWORD>(header.size()));
return SUCCEEDED(result) ? WriteAll(data, size) : result;
}
HRESULT WriteStatus(const std::string& value) {
return Write(OutputKind::status, 0, 0, 0, reinterpret_cast<const BYTE*>(value.data()),
static_cast<DWORD>(value.size()));
}
HRESULT WriteBegin(UINT64 timestamp100ns, UINT64 duration100ns) {
return Write(OutputKind::begin, 0, timestamp100ns, duration100ns, nullptr, 0);
}
HRESULT WriteUnavailable(UINT8 layer, std::string detail) {
detail.resize(std::min<size_t>(detail.size(), kMaxStatusBytes));
for (char& value : detail) {
if (static_cast<unsigned char>(value) < 32 || static_cast<unsigned char>(value) > 126) value = '?';
}
if (detail.empty()) detail = "Output encoder is unavailable";
return Write(OutputKind::unavailable, 0, 0, 0, reinterpret_cast<const BYTE*>(detail.data()),
static_cast<DWORD>(detail.size()), layer);
}
private:
static void PutBE(BYTE* output, UINT64 value, int size) {
for (int index = size - 1; index >= 0; --index) {
output[index] = static_cast<BYTE>(value & 0xff);
value >>= 8;
}
}
HRESULT WriteAll(const BYTE* data, DWORD size) {
while (size > 0) {
if (stopped_.load()) return HRESULT_FROM_WIN32(ERROR_OPERATION_ABORTED);
DWORD written = 0;
if (!WriteFile(output_, data, size, &written, nullptr)) return HRESULT_FROM_WIN32(GetLastError());
if (written == 0) return HRESULT_FROM_WIN32(ERROR_WRITE_FAULT);
data += written;
size -= written;
}
return S_OK;
}
const HANDLE output_;
std::mutex mutex_;
std::atomic<bool> stopped_{false};
};
} // namespace screener::capture
native/capture/windows/encoded_group.fixture.cpp
+87
View File
@@ -0,0 +1,87 @@
#include "vp8_encoder.h"
#include <algorithm>
#include <array>
#include <chrono>
#include <cstdint>
#include <iostream>
#include <memory>
#include <stdexcept>
#include <vector>
namespace {
using Clock = std::chrono::steady_clock;
using screener::capture::Vp8Encoder;
struct Profile { uint32_t width, height, bitrate; };
// Codec fixtures, not product presets. Layers are L=0, M=1, H=2.
constexpr std::array<Profile, 3> kProfiles{{
{160, 90, 90'000}, {320, 180, 300'000}, {640, 360, 1'200'000}}};
// Eight source frames per pair: H/H, H/L, L/L, M/M, H/H.
constexpr std::array<std::array<int, 2>, 5> kTargets{{
{2, 2}, {2, 0}, {0, 0}, {1, 1}, {2, 2}}};
void DrawFrame(std::vector<uint8_t>& pixels, Profile profile, uint32_t frame) {
pixels.assign(static_cast<size_t>(profile.width) * profile.height * 3 / 2, 128);
for (uint32_t y = 0; y < profile.height; ++y) {
for (uint32_t x = 0; x < profile.width; ++x) {
const uint32_t scene_x = (x * 640 / profile.width + frame * 8) % 640;
const uint32_t scene_y = y * 360 / profile.height;
pixels[static_cast<size_t>(y) * profile.width + x] =
static_cast<uint8_t>(32 + (((scene_x / 16) ^ (scene_y / 16)) % 8) * 24);
}
}
}
void WriteFrame(uint32_t index, int layer, Profile profile, uint64_t pts,
const screener::capture::Vp8Frame& packet) {
std::cout << "{\"FrameIndex\":" << index << ",\"Layer\":" << layer
<< ",\"Width\":" << profile.width << ",\"Height\":" << profile.height
<< ",\"PTS\":" << pts << ",\"Recovery\":"
<< (packet.key_frame ? "true" : "false") << ",\"DataHex\":\"";
constexpr char hex[] = "0123456789abcdef";
for (uint8_t byte : packet.bytes) {
std::cout.put(hex[byte >> 4]);
std::cout.put(hex[byte & 15]);
}
std::cout << "\"}\n";
if (!std::cout) throw std::runtime_error("Fixture output failed");
}
} // namespace
int main() {
try {
const auto deadline = Clock::now() + std::chrono::seconds(15);
std::array<std::unique_ptr<Vp8Encoder>, 3> encoders;
std::vector<uint8_t> pixels;
for (uint32_t frame = 0; frame < 40; ++frame) {
const auto& targets = kTargets[frame / 8];
const int highest = std::max(targets[0], targets[1]);
const uint64_t timestamp100ns = (static_cast<uint64_t>(frame) + 1) * 10'000'000 / 30;
// Serial CPU fixture only; this does not implement production worker isolation.
for (int layer = 2; layer >= 0; --layer) {
if (Clock::now() >= deadline) throw std::runtime_error("Fixture deadline exceeded");
if (layer > highest) {
encoders[layer].reset();
continue;
}
const auto profile = kProfiles[layer];
const bool recovery = frame % 8 == 0 || !encoders[layer];
if (!encoders[layer]) {
encoders[layer] = std::make_unique<Vp8Encoder>(
profile.width, profile.height, 30, profile.bitrate);
}
DrawFrame(pixels, profile, frame);
const auto packet = encoders[layer]->Encode(
pixels.data(), profile.width, timestamp100ns, recovery);
if (Clock::now() >= deadline) throw std::runtime_error("Fixture deadline exceeded");
WriteFrame(frame, layer, profile, timestamp100ns * 100, packet);
}
}
return 0;
} catch (const std::exception& error) {
std::cerr << error.what() << '\n';
return 1;
}
}
native/capture/windows/encoded_variants.probe.cpp
+276
View File
@@ -0,0 +1,276 @@
#include "vp8_encoder.h"
#include <algorithm>
#include <array>
#include <chrono>
#include <cmath>
#include <ctime>
#include <iomanip>
#include <iostream>
#include <memory>
#include <numeric>
#include <stdexcept>
#include <string>
#include <thread>
#include <vector>
#ifdef _WIN32
#define NOMINMAX
#include <windows.h>
#include <psapi.h>
#endif
using Clock = std::chrono::steady_clock;
using screener::capture::Vp8Encoder;
namespace {
constexpr int kFps = 30;
constexpr int kClipFrames = 32;
constexpr double kFrameMs = 1000.0 / kFps;
// Measurement fixtures, not an application bitrate ladder.
struct Profile { unsigned width, height, bitrate; };
constexpr std::array<Profile, 3> kProfiles{{
{480, 270, 312'500}, {960, 540, 1'250'000}, {1920, 1080, 5'000'000}}};
using Clip = std::vector<std::vector<uint8_t>>;
double Milliseconds(Clock::duration duration) {
return std::chrono::duration<double, std::milli>(duration).count();
}
double CpuMilliseconds() {
#ifdef _WIN32
FILETIME created{}, exited{}, kernel{}, user{};
if (!GetProcessTimes(GetCurrentProcess(), &created, &exited, &kernel, &user)) {
throw std::runtime_error("GetProcessTimes failed");
}
auto ticks = [](FILETIME value) {
return (static_cast<uint64_t>(value.dwHighDateTime) << 32) | value.dwLowDateTime;
};
return static_cast<double>(ticks(kernel) + ticks(user)) / 10'000;
#else
return 1000.0 * std::clock() / CLOCKS_PER_SEC;
#endif
}
uint64_t CpuCycles() {
#ifdef _WIN32
ULONG64 cycles = 0;
if (!QueryProcessCycleTime(GetCurrentProcess(), &cycles)) {
throw std::runtime_error("QueryProcessCycleTime failed");
}
return cycles;
#else
return 0;
#endif
}
size_t ResidentBytes() {
#ifdef _WIN32
PROCESS_MEMORY_COUNTERS counters{};
if (!GetProcessMemoryInfo(GetCurrentProcess(), &counters, sizeof(counters))) {
throw std::runtime_error("GetProcessMemoryInfo failed");
}
return counters.WorkingSetSize;
#else
return 0;
#endif
}
Clip MakeClip(Profile profile, bool motion) {
Clip clip(kClipFrames);
for (int frame = 0; frame < kClipFrames; ++frame) {
auto& pixels = clip[frame];
pixels.assign(static_cast<size_t>(profile.width) * profile.height * 3 / 2, 128);
for (unsigned y = 0; y < profile.height; ++y) {
for (unsigned x = 0; x < profile.width; ++x) {
const unsigned shift = motion || (x < profile.width / 16 && y < profile.height / 16)
? static_cast<unsigned>(frame) * 4 : 0;
const unsigned u = (x * 1920 / profile.width + shift) % 128;
const unsigned v = (y * 1080 / profile.height) % 128;
pixels[static_cast<size_t>(y) * profile.width + x] =
static_cast<uint8_t>(32 + ((u / 8) ^ (v / 8)) * 8 + ((u ^ v) & 31) * 2);
}
}
}
return clip;
}
struct Stats {
std::vector<double> work;
double cpu_ms = 0;
uint64_t cpu_cycles = 0;
double init_ms = 0;
int creates = 0;
std::array<int, 4> encodes{};
std::array<uint64_t, 4> bytes{};
};
template <typename T, size_t N>
void PrintArray(const std::array<T, N>& values) {
std::cout << '[';
for (size_t i = 0; i < N; ++i) std::cout << (i ? "," : "") << values[i];
std::cout << ']';
}
void PrintStats(const Stats& stats) {
auto sorted = stats.work;
std::sort(sorted.begin(), sorted.end());
const double media_ms = static_cast<double>(sorted.size()) * kFrameMs;
std::cout << "{\"sourceFrames\":" << sorted.size()
<< ",\"workMeanMs\":" << std::accumulate(sorted.begin(), sorted.end(), 0.0) / sorted.size()
<< ",\"workP95Ms\":" << sorted[static_cast<size_t>(std::ceil(sorted.size() * 0.95)) - 1]
<< ",\"workOverFrameBudget\":" << std::count_if(sorted.begin(), sorted.end(), [](double value) { return value > kFrameMs; })
<< ",\"cpuMs\":" << stats.cpu_ms
<< ",\"cpuCyclesPerFrame\":" << static_cast<double>(stats.cpu_cycles) / sorted.size()
<< ",\"cpuCoresAt30Fps\":" << stats.cpu_ms / media_ms
<< ",\"createCount\":" << stats.creates << ",\"createMs\":" << stats.init_ms
<< ",\"encodedFrames\":";
PrintArray(stats.encodes);
std::cout << ",\"encodedBytes\":";
PrintArray(stats.bytes);
std::cout << '}';
}
struct Transition { int frame, layer; double ready_ms; size_t bytes; };
void Run(const std::string& scenario, const std::string& scene) {
const bool warm = scenario == "exact-warm" || scenario == "warm-clock-ablation";
const bool trace = scenario == "envelope" || scenario == "exact-cold" || warm;
unsigned fixed_mask = 0;
if (scenario == "high") fixed_mask = 4;
if (scenario == "two-high") fixed_mask = 12;
if (scenario == "high-low") fixed_mask = 5;
if (scenario == "all") fixed_mask = 7;
if (scenario == "mid-low") fixed_mask = 3;
if (scenario == "low") fixed_mask = 1;
if ((!trace && fixed_mask == 0) || (scene != "motion" && scene != "still")) {
throw std::runtime_error("Unknown scenario or scene");
}
// Same scene at each size; preparation is excluded, no scaler is being measured.
std::array<Clip, 3> clips;
for (size_t i = 0; i < clips.size(); ++i) clips[i] = MakeClip(kProfiles[i], scene == "motion");
const size_t input_resident = ResidentBytes();
std::array<std::unique_ptr<Vp8Encoder>, 4> encoders;
std::array<uint64_t, 4> codec_frames{};
constexpr std::array<std::array<int, 2>, 7> demands{{
{2, 2}, {2, 0}, {0, 0}, {1, 1}, {2, 2}, {2, 0}, {2, 2}}};
const int frames = trace ? static_cast<int>(demands.size()) * 60 : 120;
std::array<int, 2> previous_demand{-1, -1};
unsigned previous_active = 0;
std::vector<Transition> transitions;
std::vector<Stats> phases(trace ? demands.size() : 1);
Stats total;
const auto started = Clock::now();
double phase_cpu_start = CpuMilliseconds();
uint64_t phase_cycles_start = CpuCycles();
for (int frame = 0; frame < frames; ++frame) {
if (Clock::now() - started > std::chrono::seconds(30)) throw std::runtime_error("Probe deadline exceeded");
const size_t phase = trace ? static_cast<size_t>(frame / 60) : 0;
unsigned active = fixed_mask;
unsigned switched = 0;
if (trace) {
const auto& demand = demands[phase];
active = (1u << demand[0]) | (1u << demand[1]);
if (scenario == "envelope") active = (1u << (std::max(demand[0], demand[1]) + 1)) - 1;
for (size_t child = 0; child < demand.size(); ++child) {
if (demand[child] != previous_demand[child]) switched |= 1u << demand[child];
}
previous_demand = demand;
}
const bool measured = trace || frame >= 30;
const auto tick = Clock::now();
Stats step;
// Serial scheduling keeps the probe bounded; parallel encoder scheduling is not measured.
for (int slot = 3; slot >= 0; --slot) {
const unsigned bit = 1u << slot;
if (!(active & bit)) {
if (!warm) encoders[slot].reset();
continue;
}
const size_t layer = static_cast<size_t>(std::min(slot, 2));
const auto profile = kProfiles[layer];
if (!encoders[slot]) {
const auto init = Clock::now();
encoders[slot] = std::make_unique<Vp8Encoder>(profile.width, profile.height, kFps, profile.bitrate);
step.init_ms += Milliseconds(Clock::now() - init);
++step.creates;
}
const bool keyframe = !(previous_active & bit) || (switched & bit);
// Isolate codec timestamp-gap effects only; not a proposed media clock policy.
const uint64_t pts_frame = scenario == "warm-clock-ablation" ? codec_frames[slot] : static_cast<uint64_t>(frame);
auto packet = encoders[slot]->Encode(clips[layer][frame % kClipFrames].data(), profile.width,
pts_frame * 10'000'000 / kFps, keyframe);
++codec_frames[slot];
++step.encodes[slot];
step.bytes[slot] += packet.bytes.size();
if (switched & bit) {
if (!packet.key_frame) throw std::runtime_error("Switch lacked a keyframe");
transitions.push_back({frame, slot, Milliseconds(Clock::now() - tick), packet.bytes.size()});
}
}
const double work_ms = Milliseconds(Clock::now() - tick);
if (measured) {
for (Stats* stats : {&total, &phases[phase]}) {
stats->work.push_back(work_ms);
stats->init_ms += step.init_ms;
stats->creates += step.creates;
for (size_t i = 0; i < step.encodes.size(); ++i) {
stats->encodes[i] += step.encodes[i];
stats->bytes[i] += step.bytes[i];
}
}
}
previous_active = active;
std::this_thread::sleep_until(started + std::chrono::microseconds(static_cast<int64_t>(frame + 1) * 1'000'000 / kFps));
// Whole windows include worker spin/wakeup cost and avoid per-call CPU tick quantization.
if (!trace && frame == 29) {
phase_cpu_start = CpuMilliseconds();
phase_cycles_start = CpuCycles();
}
if ((trace && (frame + 1) % 60 == 0) || frame + 1 == frames) {
const double cpu_now = CpuMilliseconds();
const uint64_t cycles_now = CpuCycles();
phases[phase].cpu_ms = cpu_now - phase_cpu_start;
phases[phase].cpu_cycles = cycles_now - phase_cycles_start;
total.cpu_ms += phases[phase].cpu_ms;
total.cpu_cycles += phases[phase].cpu_cycles;
phase_cpu_start = cpu_now;
phase_cycles_start = cycles_now;
}
}
const size_t active_resident = ResidentBytes();
for (auto& encoder : encoders) encoder.reset();
const size_t stopped_resident = ResidentBytes();
std::cout << std::fixed << std::setprecision(3)
<< "{\"scenario\":\"" << scenario << "\",\"scene\":\"" << scene
<< "\",\"codec\":\"" << Vp8Encoder::Version() << "\",\"summary\":";
PrintStats(total);
std::cout << ",\"inputResidentBytes\":" << input_resident << ",\"activeResidentBytes\":" << active_resident
<< ",\"stoppedResidentBytes\":" << stopped_resident << ",\"phases\":[";
for (size_t i = 0; i < phases.size(); ++i) {
if (i) std::cout << ',';
PrintStats(phases[i]);
}
std::cout << "],\"transitions\":[";
for (size_t i = 0; i < transitions.size(); ++i) {
const auto& value = transitions[i];
if (i) std::cout << ',';
std::cout << "{\"frame\":" << value.frame << ",\"layer\":" << value.layer
<< ",\"keyframeReadyMs\":" << value.ready_ms << ",\"keyframeBytes\":" << value.bytes << '}';
}
std::cout << "]}\n";
}
} // namespace
int main(int argc, char** argv) {
try {
if (argc != 3) throw std::runtime_error("Usage: encoded-variants.probe <scenario> <still|motion>");
Run(argv[1], argv[2]);
return 0;
} catch (const std::exception& error) {
std::cerr << error.what() << '\n';
return 1;
}
}
native/capture/windows/get-webrtc.ps1
+63
View File
@@ -0,0 +1,63 @@
[CmdletBinding()]
param()
Set-StrictMode -Version Latest
$ErrorActionPreference = 'Stop'
$ProgressPreference = 'SilentlyContinue'
$repositoryRoot = [IO.Path]::GetFullPath((Join-Path $PSScriptRoot '..\..\..'))
$pins = Get-Content -Raw (Join-Path $PSScriptRoot 'webrtc-dependencies.json') | ConvertFrom-Json
$cachePath = Join-Path $repositoryRoot 'build\encoder-pool'
$sdkPath = Join-Path $cachePath 'webrtc'
$toolchainPath = Join-Path $cachePath 'msvc-crt'
New-Item -ItemType Directory -Path $cachePath,$toolchainPath -Force | Out-Null
function Expand-PinnedArchive($pin, [string]$name, [string]$destination, [string]$entry) {
$archive = Join-Path $cachePath $name
if (!(Test-Path $archive) -or
(Get-FileHash $archive -Algorithm SHA256).Hash.ToLowerInvariant() -ne $pin.sha256) {
$download = $archive + '.download'
Invoke-WebRequest -UseBasicParsing -Uri $pin.url -OutFile $download -TimeoutSec 600
if ((Get-FileHash $download -Algorithm SHA256).Hash.ToLowerInvariant() -ne $pin.sha256) {
throw "Native WebRTC dependency digest mismatch: $name"
}
Move-Item -LiteralPath $download -Destination $archive -Force
}
$stamp = $archive + '.extracted'
if (!(Test-Path (Join-Path $destination $entry)) -or !(Test-Path $stamp) -or
(Get-Content -Raw $stamp).Trim() -ne $pin.sha256) {
& tar -xf $archive -C $destination $entry
if ($LASTEXITCODE -ne 0) { throw "Could not unpack native WebRTC dependency: $name" }
[IO.File]::WriteAllText($stamp, $pin.sha256 + "`n")
}
}
Expand-PinnedArchive $pins.webrtc 'webrtc.windows_x86_64.zip' $cachePath 'webrtc'
Expand-PinnedArchive $pins.msvc.crt 'msvc-crt.vsix' $toolchainPath ($pins.msvc.directory + '/lib/x64')
Expand-PinnedArchive $pins.msvc.tools 'msvc-tools.vsix' $toolchainPath ($pins.msvc.directory + '/bin/Hostx64/x64')
$versions = Get-Content -Raw (Join-Path $sdkPath 'VERSIONS') | ConvertFrom-StringData
if ($versions.WEBRTC_COMMIT -ne $pins.webrtc.commit -or
('m' + $versions.WEBRTC_BUILD_VERSION) -ne $pins.webrtc.version) {
throw 'Cached WebRTC SDK version differs from the dependency pin.'
}
$notices = Get-Content -Raw (Join-Path $repositoryRoot 'licenses\upstream.json') | ConvertFrom-Json
$notice = $notices.('webrtc@' + $pins.webrtc.version)
$packagedNotice = Join-Path $repositoryRoot ('licenses\' + $notice.file)
# Tracked notices use LF without trailing whitespace; the Windows SDK uses CRLF.
$sdkNotice = [IO.File]::ReadAllText((Join-Path $sdkPath 'NOTICE')).Replace("`r`n", "`n").TrimEnd() + "`n"
$sdkNotice = [regex]::Replace($sdkNotice, '(?m)[ \t]+$', '')
$sha256 = [Security.Cryptography.SHA256]::Create()
try {
$sdkNoticeHash = [BitConverter]::ToString($sha256.ComputeHash([Text.Encoding]::UTF8.GetBytes($sdkNotice))).Replace('-', '').ToLowerInvariant()
} finally { $sha256.Dispose() }
if ($sdkNoticeHash -ne $notice.sha256 -or
(Get-FileHash $packagedNotice -Algorithm SHA256).Hash.ToLowerInvariant() -ne $notice.sha256) {
throw 'WebRTC SDK and packaged notices must match the pinned upstream notice.'
}
$msvcPath = Join-Path $toolchainPath $pins.msvc.directory
return @{
Include = Join-Path $sdkPath 'include'
Library = Join-Path $sdkPath 'lib\webrtc.lib'
RuntimeLibraries = Join-Path $msvcPath 'lib\x64'
Linker = Join-Path $msvcPath 'bin\Hostx64\x64\link.exe'
}
native/capture/windows/h264_decoder.h
+171
View File
@@ -0,0 +1,171 @@
#pragma once
#include "capture_input.h"
#include <codecapi.h>
#include <mfapi.h>
#include <mferror.h>
#include <mfidl.h>
#include <mftransform.h>
#include <wmcodecdsp.h>
#include <wrl/client.h>
#include <algorithm>
#include <functional>
#include <stdexcept>
#include <string>
#include <utility>
namespace screener::capture {
// One inbox decoder owns the compressed reference chain. Consumers must finish
// copying/retaining an output sample before the callback returns.
class H264Decoder final {
public:
using Consumer = std::function<void(IMFSample*, UINT32, UINT32, LONG)>;
explicit H264Decoder(IMFDXGIDeviceManager* manager) {
Check(CoCreateInstance(__uuidof(CMSH264DecoderMFT), nullptr, CLSCTX_INPROC_SERVER,
IID_PPV_ARGS(&decoder_)), "create");
Microsoft::WRL::ComPtr<IMFAttributes> attributes;
Check(decoder_->GetAttributes(&attributes), "attributes");
UINT32 aware = FALSE;
if (manager && SUCCEEDED(attributes->GetUINT32(MF_SA_D3D11_AWARE, &aware)) && aware) {
Check(decoder_->ProcessMessage(MFT_MESSAGE_SET_D3D_MANAGER,
reinterpret_cast<ULONG_PTR>(manager)), "device-manager");
}
Microsoft::WRL::ComPtr<ICodecAPI> codec;
Check(decoder_.As(&codec), "codec-api");
for (const auto& setting : {
std::pair<GUID, ULONG>{CODECAPI_AVLowLatencyMode, TRUE},
std::pair<GUID, ULONG>{CODECAPI_AVDecVideoMaxCodedWidth, 2560},
std::pair<GUID, ULONG>{CODECAPI_AVDecVideoMaxCodedHeight, 1440}}) {
VARIANT value;
VariantInit(&value);
value.vt = VT_UI4;
value.ulVal = setting.second;
Check(codec->SetValue(&setting.first, &value), "codec-setting");
}
}
~H264Decoder() {
if (configured_) (void)decoder_->ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0);
}
H264Decoder(const H264Decoder&) = delete;
H264Decoder& operator=(const H264Decoder&) = delete;
void Decode(const InputEnvelope& input, const Consumer& consume) {
if (input.kind != 2 || input.data.empty() || input.data.size() > 4 * 1024 * 1024)
throw std::runtime_error("Invalid H264 decoder input");
if (!configured_) {
if (!input.recovery) throw std::runtime_error("H264 decoder needs an independent recovery input");
Microsoft::WRL::ComPtr<IMFMediaType> type;
Check(MFCreateMediaType(&type), "input-type");
Check(type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video), "input-major");
Check(type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264), "input-subtype");
Check(MFSetAttributeSize(type.Get(), MF_MT_FRAME_SIZE, input.width, input.height), "input-size");
Check(type->SetUINT32(MF_MT_INTERLACE_MODE, MFVideoInterlace_Progressive), "input-progressive");
Check(decoder_->SetInputType(0, type.Get(), 0), "set-input");
SelectOutput();
Check(decoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0), "begin");
Check(decoder_->ProcessMessage(MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0), "start");
configured_ = true;
}
Microsoft::WRL::ComPtr<IMFMediaBuffer> buffer;
Check(MFCreateMemoryBuffer(static_cast<DWORD>(input.data.size()), &buffer), "input-buffer");
BYTE* bytes = nullptr;
Check(buffer->Lock(&bytes, nullptr, nullptr), "input-lock");
std::copy(input.data.begin(), input.data.end(), bytes);
Check(buffer->Unlock(), "input-unlock");
Check(buffer->SetCurrentLength(static_cast<DWORD>(input.data.size())), "input-length");
Microsoft::WRL::ComPtr<IMFSample> sample;
Check(MFCreateSample(&sample), "input-sample");
Check(sample->AddBuffer(buffer.Get()), "input-sample-buffer");
Check(sample->SetSampleTime(static_cast<LONGLONG>(input.timestamp)), "input-timestamp");
Check(sample->SetSampleDuration(static_cast<LONGLONG>(input.duration)), "input-duration");
HRESULT result = decoder_->ProcessInput(0, sample.Get(), 0);
if (result == MF_E_NOTACCEPTING) {
Drain(consume);
result = decoder_->ProcessInput(0, sample.Get(), 0);
}
Check(result, "process-input");
Drain(consume);
}
private:
static void Check(HRESULT result, const char* operation) {
if (FAILED(result)) throw std::runtime_error(std::string("H264 decoder ") + operation + " failed");
}
void SelectOutput() {
for (DWORD index = 0; index < 64; ++index) {
Microsoft::WRL::ComPtr<IMFMediaType> type;
const HRESULT result = decoder_->GetOutputAvailableType(0, index, &type);
if (result == MF_E_NO_MORE_TYPES) break;
Check(result, "output-type");
GUID subtype{};
Check(type->GetGUID(MF_MT_SUBTYPE, &subtype), "output-subtype");
if (subtype != MFVideoFormat_NV12) continue;
UINT32 width = 0, height = 0;
Check(MFGetAttributeSize(type.Get(), MF_MT_FRAME_SIZE, &width, &height), "output-size");
if (width < 2 || height < 2 || width > 2560 || height > 1440 || (width & 1) || (height & 1)) {
throw std::runtime_error("H264 decoder output dimensions exceed the source bound");
}
Check(decoder_->SetOutputType(0, type.Get(), 0), "set-output");
UINT32 stride = 0;
if (FAILED(type->GetUINT32(MF_MT_DEFAULT_STRIDE, &stride))) {
LONG fallback = 0;
Check(MFGetStrideForBitmapInfoHeader(MFVideoFormat_NV12.Data1, width, &fallback), "output-stride");
stride = static_cast<UINT32>(fallback);
}
if (static_cast<LONG>(stride) < static_cast<LONG>(width)) throw std::runtime_error("H264 decoder returned an invalid NV12 stride");
width_ = width;
height_ = height;
stride_ = static_cast<LONG>(stride);
return;
}
throw std::runtime_error("H264 decoder has no NV12 output");
}
void Drain(const Consumer& consume) {
for (unsigned iteration = 0; iteration < 64; ++iteration) {
MFT_OUTPUT_STREAM_INFO info{};
Check(decoder_->GetOutputStreamInfo(0, &info), "output-info");
Microsoft::WRL::ComPtr<IMFSample> owned;
if (!(info.dwFlags & (MFT_OUTPUT_STREAM_PROVIDES_SAMPLES | MFT_OUTPUT_STREAM_CAN_PROVIDE_SAMPLES))) {
const DWORD expected = static_cast<DWORD>(stride_) * height_ * 3 / 2;
const DWORD size = std::max(info.cbSize, expected);
if (size > 2560 * 1440 * 2) throw std::runtime_error("H264 decoder output allocation exceeds its bound");
Microsoft::WRL::ComPtr<IMFMediaBuffer> buffer;
Check(MFCreateAlignedMemoryBuffer(size, info.cbAlignment ? info.cbAlignment - 1 : 0, &buffer), "output-buffer");
Check(MFCreateSample(&owned), "output-sample");
Check(owned->AddBuffer(buffer.Get()), "output-sample-buffer");
}
MFT_OUTPUT_DATA_BUFFER output{};
output.pSample = owned.Get();
DWORD status = 0;
const HRESULT result = decoder_->ProcessOutput(0, 1, &output, &status);
Microsoft::WRL::ComPtr<IMFCollection> events;
events.Attach(output.pEvents);
if (!owned) owned.Attach(output.pSample);
if (result == MF_E_TRANSFORM_NEED_MORE_INPUT) return;
if (result == MF_E_TRANSFORM_STREAM_CHANGE) {
SelectOutput();
continue;
}
Check(result, "process-output");
if (owned && !(output.dwStatus & MFT_OUTPUT_DATA_BUFFER_NO_SAMPLE)) {
consume(owned.Get(), width_, height_, stride_);
}
}
throw std::runtime_error("H264 decoder exceeded its bounded output pump");
}
Microsoft::WRL::ComPtr<IMFTransform> decoder_;
UINT32 width_ = 0;
UINT32 height_ = 0;
LONG stride_ = 0;
bool configured_ = false;
};
} // namespace screener::capture
native/capture/windows/h264_encoder.cpp
+746
View File
@@ -0,0 +1,746 @@
#include "h264_encoder.h"
#include "capture_output.h"
#include <d3d10_1.h>
#include <evr.h>
#include <mferror.h>
#include <propvarutil.h>
#include <algorithm>
#include <thread>
namespace screener::capture::windows {
namespace {
constexpr DWORD kMaxEncodedSampleBytes = 4 * 1024 * 1024;
}
[[noreturn]] void Fail(const std::string& stage, const std::string& detail) {
throw GateFailure(stage, detail);
}
void Check(HRESULT result, const std::string& stage) {
if (FAILED(result)) {
throw GateFailure(stage, "Windows API returned a failing HRESULT", result);
}
}
std::string NarrowAscii(const std::wstring& value) {
std::string result;
result.reserve(value.size());
for (wchar_t character : value) {
result.push_back(character >= 32 && character <= 126
? static_cast<char>(character)
: '?');
}
return result;
}
std::string GuidString(const GUID& value) {
wchar_t buffer[40] = {};
if (StringFromGUID2(value, buffer, static_cast<int>(std::size(buffer))) == 0) {
return "unavailable";
}
return NarrowAscii(buffer);
}
std::string HexByte(UINT8 value) {
std::ostringstream output;
output << std::hex << std::setfill('0') << std::setw(2)
<< static_cast<UINT32>(value);
return output.str();
}
std::string ProfileLevelId(UINT8 profile, UINT8 constraints, UINT8 level) {
return HexByte(profile) + HexByte(constraints) + HexByte(level);
}
std::vector<Adapter> EnumerateAdapters() {
ComPtr<IDXGIFactory1> factory;
Check(CreateDXGIFactory1(IID_PPV_ARGS(&factory)), "dxgi-factory");
std::vector<Adapter> adapters;
for (UINT index = 0;; ++index) {
ComPtr<IDXGIAdapter1> adapter;
HRESULT result = factory->EnumAdapters1(index, &adapter);
if (result == DXGI_ERROR_NOT_FOUND) {
break;
}
Check(result, "dxgi-enumerate-adapter");
Adapter current;
current.index = index;
current.value = adapter;
Check(adapter->GetDesc1(&current.description), "dxgi-adapter-description");
if ((current.description.Flags & DXGI_ADAPTER_FLAG_SOFTWARE) == 0) {
adapters.push_back(std::move(current));
}
}
return adapters;
}
const Adapter& SelectAdapter(const std::vector<Adapter>& adapters, UINT index) {
auto found = std::find_if(adapters.begin(), adapters.end(),
[index](const Adapter& adapter) {
return adapter.index == index;
});
if (found == adapters.end()) {
Fail("adapter-select", "selected DXGI adapter index is unavailable");
}
return *found;
}
DeviceContext CreateDevice(const Adapter& adapter) {
DeviceContext result;
constexpr D3D_FEATURE_LEVEL levels[] = {
D3D_FEATURE_LEVEL_12_1, D3D_FEATURE_LEVEL_12_0,
D3D_FEATURE_LEVEL_11_1, D3D_FEATURE_LEVEL_11_0,
};
Check(D3D11CreateDevice(adapter.value.Get(), D3D_DRIVER_TYPE_UNKNOWN, nullptr,
D3D11_CREATE_DEVICE_VIDEO_SUPPORT |
D3D11_CREATE_DEVICE_BGRA_SUPPORT,
levels, static_cast<UINT>(std::size(levels)),
D3D11_SDK_VERSION, &result.device,
&result.feature_level, &result.context),
"d3d11-create-device");
ComPtr<ID3D10Multithread> multithread;
Check(result.device.As(&multithread), "d3d11-multithread-interface");
multithread->SetMultithreadProtected(TRUE);
if (!multithread->GetMultithreadProtected()) {
Fail("d3d11-multithread-protection",
"D3D11 device did not enable multithread protection");
}
UINT reset_token = 0;
Check(MFCreateDXGIDeviceManager(&reset_token, &result.manager),
"dxgi-device-manager-create");
Check(result.manager->ResetDevice(result.device.Get(), reset_token),
"dxgi-device-manager-reset");
return result;
}
std::wstring GetOptionalString(IMFAttributes* attributes, const GUID& key) {
wchar_t* raw = nullptr;
UINT32 length = 0;
HRESULT result = attributes->GetAllocatedString(key, &raw, &length);
if (result == MF_E_ATTRIBUTENOTFOUND) {
return L"";
}
Check(result, "mft-activation-string");
std::wstring value(raw, length);
CoTaskMemFree(raw);
return value;
}
ActivationList EnumerateHardwareEncoders(const Adapter& adapter,
bool require_candidate) {
ComPtr<IMFAttributes> attributes;
Check(MFCreateAttributes(&attributes, 1), "mft-enumeration-attributes");
Check(attributes->SetBlob(MFT_ENUM_ADAPTER_LUID,
reinterpret_cast<const UINT8*>(
&adapter.description.AdapterLuid),
sizeof(LUID)),
"mft-enumeration-adapter-luid");
MFT_REGISTER_TYPE_INFO input = {MFMediaType_Video, MFVideoFormat_NV12};
MFT_REGISTER_TYPE_INFO output = {MFMediaType_Video, MFVideoFormat_H264};
ActivationList list;
Check(MFTEnum2(MFT_CATEGORY_VIDEO_ENCODER,
MFT_ENUM_FLAG_HARDWARE | MFT_ENUM_FLAG_SORTANDFILTER,
&input, &output, attributes.Get(), &list.values, &list.count),
"mft-hardware-enumeration");
if (require_candidate && list.count == 0) {
Fail("mft-hardware-enumeration",
"selected adapter exposed no hardware NV12-to-H264 MFT");
}
return list;
}
SelectedTransform ActivateTransform(const ActivationList& list, UINT index,
IMFDXGIDeviceManager* manager) {
if (index >= list.count) {
Fail("mft-select", "selected hardware MFT index is unavailable");
}
IMFActivate* activation = list.values[index];
std::wstring friendly = GetOptionalString(activation, MFT_FRIENDLY_NAME_Attribute);
if (friendly.empty()) {
Fail("mft-identity", "hardware MFT has no friendly-name attribute");
}
GUID clsid = {};
Check(activation->GetGUID(MFT_TRANSFORM_CLSID_Attribute, &clsid),
"mft-clsid");
PROPVARIANT hardware_url;
PropVariantInit(&hardware_url);
HRESULT hardware_result = activation->GetItem(
MFT_ENUM_HARDWARE_URL_Attribute, &hardware_url);
PropVariantClear(&hardware_url);
if (FAILED(hardware_result)) {
Fail("mft-hardware-identity",
"hardware-only activation omitted its hardware URL attribute");
}
SelectedTransform selected;
selected.activation = activation;
selected.name = NarrowAscii(friendly);
selected.clsid = GuidString(clsid);
Check(activation->ActivateObject(IID_PPV_ARGS(&selected.transform)),
"mft-activate");
ComPtr<IMFAttributes> transform_attributes;
Check(selected.transform->GetAttributes(&transform_attributes),
"mft-global-attributes");
UINT32 is_async = 0;
Check(transform_attributes->GetUINT32(MF_TRANSFORM_ASYNC, &is_async),
"mft-async-readback");
if (is_async != TRUE) {
Fail("mft-async-readback", "hardware MFT is not asynchronous");
}
Check(transform_attributes->SetUINT32(MF_TRANSFORM_ASYNC_UNLOCK, TRUE),
"mft-async-unlock");
UINT32 d3d11_aware = 0;
Check(transform_attributes->GetUINT32(MF_SA_D3D11_AWARE, &d3d11_aware),
"mft-d3d11-aware-readback");
if (d3d11_aware != TRUE) {
Fail("mft-d3d11-aware-readback", "hardware MFT is not D3D11-aware");
}
Check(selected.transform->ProcessMessage(
MFT_MESSAGE_SET_D3D_MANAGER,
reinterpret_cast<ULONG_PTR>(manager)),
"mft-set-d3d-manager");
Check(selected.transform.As(&selected.codec), "mft-codec-api");
Check(selected.transform.As(&selected.events), "mft-event-generator");
return selected;
}
void RequireProperty(ICodecAPI* codec, const GUID& key,
const std::string& name) {
HRESULT supported = codec->IsSupported(&key);
if (supported != S_OK) {
if (FAILED(supported) && supported != E_NOTIMPL) {
Check(supported, name + "-supported-query");
}
Fail(name + "-unsupported", "required codec property is unsupported");
}
HRESULT modifiable = codec->IsModifiable(&key);
if (modifiable == S_FALSE) {
Fail(name + "-readonly", "required codec property is read-only");
}
// Some hardware MFTs do not implement this advisory query. SetValue=S_OK
// followed by exact GetValue readback remains the authoritative gate.
if (modifiable != S_OK && modifiable != E_NOTIMPL) {
Check(modifiable, name + "-modifiable-query");
}
}
UINT32 ReadU32(ICodecAPI* codec, const GUID& key, const std::string& name) {
VARIANT value;
VariantInit(&value);
Check(codec->GetValue(&key, &value), name + "-readback");
if (value.vt != VT_UI4) {
VariantClear(&value);
Fail(name + "-readback-type", "codec property did not return VT_UI4");
}
UINT32 result = value.ulVal;
VariantClear(&value);
return result;
}
bool ReadBool(ICodecAPI* codec, const GUID& key, const std::string& name) {
VARIANT value;
VariantInit(&value);
Check(codec->GetValue(&key, &value), name + "-readback");
if (value.vt != VT_BOOL) {
VariantClear(&value);
Fail(name + "-readback-type", "codec property did not return VT_BOOL");
}
bool result = value.boolVal != VARIANT_FALSE;
VariantClear(&value);
return result;
}
void SetU32(ICodecAPI* codec, const GUID& key, UINT32 expected,
const std::string& name) {
RequireProperty(codec, key, name);
VARIANT value;
VariantInit(&value);
value.vt = VT_UI4;
value.ulVal = expected;
HRESULT set_result = codec->SetValue(&key, &value);
VariantClear(&value);
if (set_result != S_OK) {
if (FAILED(set_result)) {
Check(set_result, name + "-set");
}
Fail(name + "-readonly", "codec property rejected the requested value");
}
if (ReadU32(codec, key, name) != expected) {
Fail(name + "-weakened", "codec property readback changed the requested value");
}
}
void SetBool(ICodecAPI* codec, const GUID& key, bool expected,
const std::string& name) {
RequireProperty(codec, key, name);
VARIANT value;
VariantInit(&value);
value.vt = VT_BOOL;
value.boolVal = expected ? VARIANT_TRUE : VARIANT_FALSE;
HRESULT set_result = codec->SetValue(&key, &value);
VariantClear(&value);
if (set_result != S_OK) {
if (FAILED(set_result)) {
Check(set_result, name + "-set");
}
Fail(name + "-readonly", "codec property rejected the requested value");
}
if (ReadBool(codec, key, name) != expected) {
Fail(name + "-weakened", "codec property readback changed the requested value");
}
}
void SetOptionalU32(ICodecAPI* codec, const GUID& key, UINT32 value) {
if (codec->IsSupported(&key) != S_OK) return;
VARIANT requested;
VariantInit(&requested);
requested.vt = VT_UI4;
requested.ulVal = value;
(void)codec->SetValue(&key, &requested);
VariantClear(&requested);
}
void ConfigureCodec(
ICodecAPI* codec,
const VideoProfile& profile) {
SetU32(codec, CODECAPI_AVEncCommonRateControlMode,
eAVEncCommonRateControlMode_CBR, "codec-cbr");
SetBool(codec, CODECAPI_AVLowLatencyMode, true, "codec-low-latency");
SetU32(codec, CODECAPI_AVEncCommonMeanBitRate, profile.bit_rate,
"codec-mean-bitrate");
SetU32(codec, CODECAPI_AVEncMPVGOPSize, profile.gop_frames(), "codec-gop");
SetOptionalU32(codec, CODECAPI_AVEncCommonQualityVsSpeed,
profile.quality_vs_speed());
RequireProperty(codec, CODECAPI_AVEncVideoForceKeyFrame,
"codec-force-keyframe");
}
ComPtr<IMFMediaType> CreateOutputType(
const VideoProfile& profile) {
ComPtr<IMFMediaType> type;
Check(MFCreateMediaType(&type), "output-type-create");
Check(type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video),
"output-type-major");
Check(type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_H264),
"output-type-subtype");
Check(type->SetUINT32(MF_MT_AVG_BITRATE, profile.bit_rate),
"output-type-bitrate");
Check(MFSetAttributeSize(type.Get(), MF_MT_FRAME_SIZE, profile.width,
profile.height),
"output-type-size");
Check(MFSetAttributeRatio(type.Get(), MF_MT_FRAME_RATE, profile.frame_rate, 1),
"output-type-framerate");
Check(MFSetAttributeRatio(type.Get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1),
"output-type-pixel-aspect");
Check(type->SetUINT32(MF_MT_INTERLACE_MODE,
MFVideoInterlace_Progressive),
"output-type-interlace");
Check(type->SetUINT32(MF_MT_MPEG2_PROFILE, eAVEncH264VProfile_Base),
"output-type-profile");
Check(type->SetUINT32(MF_MT_MPEG2_LEVEL, profile.h264_level()),
"output-type-level");
return type;
}
ComPtr<IMFMediaType> CreateInputType(
const VideoProfile& profile) {
ComPtr<IMFMediaType> type;
Check(MFCreateMediaType(&type), "input-type-create");
Check(type->SetGUID(MF_MT_MAJOR_TYPE, MFMediaType_Video),
"input-type-major");
Check(type->SetGUID(MF_MT_SUBTYPE, MFVideoFormat_NV12),
"input-type-subtype");
Check(MFSetAttributeSize(type.Get(), MF_MT_FRAME_SIZE, profile.width,
profile.height),
"input-type-size");
Check(MFSetAttributeRatio(type.Get(), MF_MT_FRAME_RATE, profile.frame_rate, 1),
"input-type-framerate");
Check(MFSetAttributeRatio(type.Get(), MF_MT_PIXEL_ASPECT_RATIO, 1, 1),
"input-type-pixel-aspect");
Check(type->SetUINT32(MF_MT_INTERLACE_MODE,
MFVideoInterlace_Progressive),
"input-type-interlace");
Check(type->SetUINT32(MF_MT_DEFAULT_STRIDE, profile.width),
"input-type-stride");
return type;
}
void RequireTypeU32(IMFAttributes* type, const GUID& key, UINT32 expected,
const std::string& stage) {
UINT32 actual = 0;
Check(type->GetUINT32(key, &actual), stage + "-readback");
if (actual != expected) {
Fail(stage + "-weakened", "media type changed a required value");
}
}
void ValidateMediaTypes(
IMFTransform* transform,
const VideoProfile& profile) {
ComPtr<IMFMediaType> output;
Check(transform->GetOutputCurrentType(0, &output),
"output-type-current");
GUID subtype = {};
Check(output->GetGUID(MF_MT_SUBTYPE, &subtype), "output-type-subtype-readback");
if (subtype != MFVideoFormat_H264) {
Fail("output-type-subtype-weakened", "output is not H264");
}
RequireTypeU32(output.Get(), MF_MT_AVG_BITRATE, profile.bit_rate,
"output-type-bitrate");
RequireTypeU32(output.Get(), MF_MT_MPEG2_PROFILE,
eAVEncH264VProfile_Base, "output-type-profile");
RequireTypeU32(output.Get(), MF_MT_MPEG2_LEVEL, profile.h264_level(),
"output-type-level");
UINT32 width = 0;
UINT32 height = 0;
Check(MFGetAttributeSize(output.Get(), MF_MT_FRAME_SIZE, &width, &height),
"output-type-size-readback");
if (width != profile.width || height != profile.height) {
Fail("output-type-size-weakened", "output dimensions changed");
}
UINT32 numerator = 0;
UINT32 denominator = 0;
Check(MFGetAttributeRatio(output.Get(), MF_MT_FRAME_RATE, &numerator,
&denominator),
"output-type-framerate-readback");
if (numerator != profile.frame_rate || denominator != 1) {
Fail("output-type-framerate-weakened", "output frame rate changed");
}
}
void ValidateCodecReadback(
ICodecAPI* codec,
const VideoProfile& profile) {
if (ReadU32(codec, CODECAPI_AVEncCommonRateControlMode, "codec-cbr") !=
eAVEncCommonRateControlMode_CBR ||
!ReadBool(codec, CODECAPI_AVLowLatencyMode, "codec-low-latency") ||
ReadU32(codec, CODECAPI_AVEncCommonMeanBitRate,
"codec-mean-bitrate") != profile.bit_rate ||
ReadU32(codec, CODECAPI_AVEncMPVGOPSize, "codec-gop") !=
profile.gop_frames()) {
Fail("codec-final-readback", "one or more codec properties were weakened");
}
}
std::optional<std::pair<size_t, size_t>> FindStartCode(
const std::vector<UINT8>& bytes, size_t from) {
for (size_t index = from; index + 3 <= bytes.size(); ++index) {
if (bytes[index] != 0 || bytes[index + 1] != 0) {
continue;
}
if (bytes[index + 2] == 1) {
return std::pair<size_t, size_t>{index, 3};
}
if (index + 4 <= bytes.size() && bytes[index + 2] == 0 &&
bytes[index + 3] == 1) {
return std::pair<size_t, size_t>{index, 4};
}
}
return std::nullopt;
}
NalSummary InspectAnnexB(const std::vector<UINT8>& bytes) {
NalSummary summary;
std::optional<std::pair<size_t, size_t>> current = FindStartCode(bytes, 0);
if (!current || current->first != 0) {
return summary;
}
summary.annex_b = true;
while (current) {
size_t nal_start = current->first + current->second;
std::optional<std::pair<size_t, size_t>> next = FindStartCode(bytes, nal_start);
size_t nal_end = next ? next->first : bytes.size();
if (nal_start < nal_end) {
UINT8 nal_type = bytes[nal_start] & 0x1f;
summary.sps = summary.sps || nal_type == 7;
summary.pps = summary.pps || nal_type == 8;
summary.idr = summary.idr || nal_type == 5;
if (nal_type == 7 && nal_end - nal_start >= 4) {
std::string value = ProfileLevelId(bytes[nal_start + 1],
bytes[nal_start + 2],
bytes[nal_start + 3]);
if (summary.profile_level_id && *summary.profile_level_id != value) {
Fail("bitstream-sps-consistency",
"one access unit contained conflicting SPS profile bytes");
}
summary.profile_level_id = value;
}
}
current = next;
}
return summary;
}
std::vector<UINT8> ReadSample(IMFSample* sample) {
DWORD total_length = 0;
Check(sample->GetTotalLength(&total_length), "output-total-length");
if (total_length == 0 || total_length > kMaxEncodedSampleBytes) {
Fail("output-buffer-bounds", "encoded output size is empty or unbounded");
}
ComPtr<IMFMediaBuffer> buffer;
Check(sample->ConvertToContiguousBuffer(&buffer), "output-contiguous-buffer");
DWORD length = 0;
Check(buffer->GetCurrentLength(&length), "output-buffer-length");
if (length != total_length) {
Fail("output-buffer-length", "contiguous output length changed unexpectedly");
}
std::vector<UINT8> bytes(length);
UINT8* raw = nullptr;
DWORD maximum = 0;
DWORD current = 0;
Check(buffer->Lock(&raw, &maximum, &current), "output-buffer-lock");
if (raw == nullptr || current != length || current > kMaxEncodedSampleBytes) {
buffer->Unlock();
Fail("output-buffer-lock-bounds",
"locked output length is inconsistent or unbounded");
}
std::copy(raw, raw + current, bytes.begin());
Check(buffer->Unlock(), "output-buffer-unlock");
return bytes;
}
void ForceKeyFrame(ICodecAPI* codec) {
VARIANT value;
VariantInit(&value);
value.vt = VT_UI4;
value.ulVal = 1;
HRESULT set_result =
codec->SetValue(&CODECAPI_AVEncVideoForceKeyFrame, &value);
VariantClear(&value);
if (set_result != S_OK) {
if (FAILED(set_result)) {
Check(set_result, "codec-force-keyframe-set");
}
Fail("codec-force-keyframe-readonly",
"force-key-frame command was not accepted");
}
}
ComPtr<IMFSample> CreateCallerOutputSample(const MFT_OUTPUT_STREAM_INFO& info) {
if (info.cbSize > kMaxEncodedSampleBytes) {
Fail("output-buffer-contract",
"hardware MFT requested an output buffer larger than four MiB");
}
ComPtr<IMFSample> sample;
Check(MFCreateSample(&sample), "output-sample-create");
ComPtr<IMFMediaBuffer> buffer;
Check(MFCreateMemoryBuffer(kMaxEncodedSampleBytes, &buffer),
"output-memory-buffer");
Check(sample->AddBuffer(buffer.Get()), "output-sample-buffer");
return sample;
}
ComPtr<IMFSample> PullOutput(IMFTransform* transform,
const MFT_OUTPUT_STREAM_INFO& info) {
bool transform_provides =
(info.dwFlags & (MFT_OUTPUT_STREAM_PROVIDES_SAMPLES |
MFT_OUTPUT_STREAM_CAN_PROVIDE_SAMPLES)) != 0;
ComPtr<IMFSample> caller_sample;
if (!transform_provides) {
caller_sample = CreateCallerOutputSample(info);
}
MFT_OUTPUT_DATA_BUFFER output = {};
output.dwStreamID = 0;
output.pSample = caller_sample.Get();
DWORD status = 0;
HRESULT result = transform->ProcessOutput(0, 1, &output, &status);
ComPtr<IMFCollection> events;
if (output.pEvents != nullptr) {
events.Attach(output.pEvents);
}
Check(result, "mft-process-output");
if ((output.dwStatus & MFT_OUTPUT_DATA_BUFFER_FORMAT_CHANGE) != 0 ||
(status & MFT_PROCESS_OUTPUT_STATUS_NEW_STREAMS) != 0) {
Fail("mft-output-format-change",
"hardware MFT changed format during the fixed run");
}
if (caller_sample) {
if (output.pSample != caller_sample.Get()) {
if (output.pSample != nullptr) {
output.pSample->Release();
}
Fail("mft-output-allocation",
"hardware MFT replaced the caller-owned output sample");
}
return caller_sample;
}
if (output.pSample == nullptr) {
Fail("mft-output-sample", "hardware MFT produced no output sample");
}
ComPtr<IMFSample> produced;
produced.Attach(output.pSample);
return produced;
}
ComPtr<IMFSample> CreateSurfaceSample(ID3D11Texture2D* texture,
LONGLONG timestamp100ns,
LONGLONG duration100ns) {
ComPtr<IMFMediaBuffer> buffer;
Check(MFCreateDXGISurfaceBuffer(IID_ID3D11Texture2D, texture, 0, FALSE,
&buffer),
"input-dxgi-buffer");
ComPtr<IMFSample> sample;
Check(MFCreateVideoSampleFromSurface(nullptr, &sample),
"input-video-sample");
Check(sample->AddBuffer(buffer.Get()), "input-sample-buffer");
Check(sample->SetSampleTime(timestamp100ns), "input-sample-time");
Check(sample->SetSampleDuration(duration100ns), "input-sample-duration");
return sample;
}
void RequireEncoderTime(EncoderClock::time_point deadline) {
if (EncoderClock::now() >= deadline) {
Fail("codec-probe-timeout", "native codec selection exceeded its time budget");
}
}
LiveEncoder::LiveEncoder(
SelectedTransform selected,
VideoProfile profile)
: VideoEncoder(OutputKind::h264, selected.name, selected.clsid),
selected_(std::move(selected)), profile_(profile) {
ConfigureCodec(selected_.codec.Get(), profile_);
ComPtr<IMFMediaType> output_type = CreateOutputType(profile_);
Check(selected_.transform->SetOutputType(0, output_type.Get(), 0),
"mft-set-output-type");
ComPtr<IMFMediaType> input_type = CreateInputType(profile_);
Check(selected_.transform->SetInputType(0, input_type.Get(), 0),
"mft-set-input-type");
ValidateMediaTypes(selected_.transform.Get(), profile_);
ValidateCodecReadback(selected_.codec.Get(), profile_);
Check(selected_.transform->GetOutputStreamInfo(0, &output_info_),
"mft-output-stream-info");
Check(selected_.transform->ProcessMessage(
MFT_MESSAGE_NOTIFY_BEGIN_STREAMING, 0),
"mft-begin-streaming");
Check(selected_.transform->ProcessMessage(
MFT_MESSAGE_NOTIFY_START_OF_STREAM, 0),
"mft-start-stream");
started_ = true;
}
LiveEncoder::~LiveEncoder() {
if (started_) {
selected_.transform->ProcessMessage(MFT_MESSAGE_COMMAND_FLUSH, 0);
selected_.transform->ProcessMessage(MFT_MESSAGE_NOTIFY_END_STREAMING, 0);
}
}
void LiveEncoder::SetBitrate(UINT32 bitrate) {
SetU32(selected_.codec.Get(), CODECAPI_AVEncCommonMeanBitRate, bitrate,
"codec-live-bitrate");
profile_.bit_rate = bitrate;
}
EncodedAccessUnit LiveEncoder::Encode(
ID3D11Texture2D* texture, UINT64 timestamp100ns, bool force_key_frame,
EncoderClock::time_point probe_deadline) {
WaitForInput(probe_deadline);
if (force_key_frame) ForceKeyFrame(selected_.codec.Get());
ComPtr<IMFSample> sample = CreateSurfaceSample(
texture, static_cast<LONGLONG>(timestamp100ns),
profile_.frame_duration_100ns());
Check(selected_.transform->ProcessInput(0, sample.Get(), 0),
"mft-process-input");
--input_requests_;
const auto deadline = std::min(probe_deadline,
EncoderClock::now() + std::chrono::seconds(2));
while (std::chrono::steady_clock::now() < deadline) {
MediaEventType type = NextEvent(deadline);
if (type == METransformNeedInput) {
++input_requests_;
continue;
}
if (type != METransformHaveOutput) continue;
ComPtr<IMFSample> output = PullOutput(selected_.transform.Get(),
output_info_);
LONGLONG output_time = -1;
Check(output->GetSampleTime(&output_time), "output-sample-time");
if (output_time != static_cast<LONGLONG>(timestamp100ns)) {
Fail("output-order", "hardware MFT changed or reordered a live timestamp");
}
std::vector<UINT8> bytes = ReadSample(output.Get());
NalSummary nal = InspectAnnexB(bytes);
if (!nal.annex_b) {
Fail("bitstream-annexb", "live output is not Annex-B H264");
}
if (nal.profile_level_id) {
if (*nal.profile_level_id != profile_.profile_level_id()) {
Fail("bitstream-profile", "hardware MFT changed the requested H.264 profile level");
}
if (profile_level_id_ && *profile_level_id_ != *nal.profile_level_id) {
Fail("bitstream-profile-change", "hardware MFT changed SPS profile");
}
profile_level_id_ = nal.profile_level_id;
}
if (force_key_frame && !(nal.sps && nal.pps && nal.idr)) {
Fail("bitstream-recovery-unit",
"requested live recovery unit lacks SPS, PPS, or IDR");
}
if (!profile_level_id_) {
Fail("bitstream-sps", "first live access unit did not carry an SPS");
}
RequireEncoderTime(probe_deadline);
return EncodedAccessUnit{timestamp100ns, nal.idr && nal.sps && nal.pps,
std::move(bytes)};
}
Fail("mft-output-timeout", "hardware MFT did not produce live output in time");
}
MediaEventType LiveEncoder::NextEvent(std::chrono::steady_clock::time_point deadline) {
while (std::chrono::steady_clock::now() < deadline) {
ComPtr<IMFMediaEvent> event;
HRESULT result = selected_.events->GetEvent(MF_EVENT_FLAG_NO_WAIT,
&event);
if (result == MF_E_NO_EVENTS_AVAILABLE) {
std::this_thread::sleep_for(std::chrono::milliseconds(1));
continue;
}
Check(result, "mft-get-event");
HRESULT status = S_OK;
Check(event->GetStatus(&status), "mft-event-status-read");
Check(status, "mft-event-status");
MediaEventType type = MEUnknown;
Check(event->GetType(&type), "mft-event-type");
if (type == MEError) Fail("mft-error-event", "hardware MFT emitted MEError");
return type;
}
Fail("mft-event-timeout", "hardware MFT did not request input in time");
}
void LiveEncoder::WaitForInput(EncoderClock::time_point probe_deadline) {
const auto deadline = std::min(probe_deadline,
EncoderClock::now() + std::chrono::seconds(2));
while (input_requests_ == 0) {
MediaEventType type = NextEvent(deadline);
if (type == METransformNeedInput) {
++input_requests_;
} else if (type == METransformHaveOutput) {
Fail("mft-unexpected-output", "hardware MFT produced output without live input");
}
}
}
} // namespace screener::capture::windows
native/capture/windows/h264_encoder.h
+253
View File
@@ -0,0 +1,253 @@
#pragma once
#ifndef NOMINMAX
#define NOMINMAX
#endif
#include <windows.h>
#include <codecapi.h>
#include <d3d11.h>
#include <dxgi1_2.h>
#include <strmif.h>
#include <mfapi.h>
#include <mfidl.h>
#include <mftransform.h>
#include <wrl/client.h>
#include <chrono>
#include <iomanip>
#include <optional>
#include <sstream>
#include <stdexcept>
#include <string>
#include <utility>
#include <vector>
namespace screener::capture {
enum class OutputKind : UINT8;
}
namespace screener::capture::windows {
using Microsoft::WRL::ComPtr;
enum class DegradationPreference {
resolution,
balanced,
framerate,
};
struct VideoProfile final {
UINT32 width = 1280;
UINT32 height = 720;
UINT32 frame_rate = 30;
UINT32 bit_rate = 3'000'000;
DegradationPreference preference = DegradationPreference::balanced;
UINT32 gop_frames() const { return frame_rate * 2; }
LONGLONG frame_duration_100ns() const {
return 10'000'000 / static_cast<LONGLONG>(frame_rate);
}
UINT32 quality_vs_speed() const {
if (preference == DegradationPreference::resolution) return 100;
if (preference == DegradationPreference::framerate) return 0;
return 50;
}
UINT32 h264_level() const {
const UINT64 macroblocks =
((static_cast<UINT64>(width) + 15) / 16) *
((static_cast<UINT64>(height) + 15) / 16);
const UINT64 macroblocks_per_second = macroblocks * frame_rate;
if (macroblocks <= 3'600 && macroblocks_per_second <= 108'000) return 31;
if (macroblocks <= 8'192 && macroblocks_per_second <= 245'760) return 40;
if (macroblocks <= 8'704 && macroblocks_per_second <= 522'240) return 42;
if (macroblocks <= 22'080 && macroblocks_per_second <= 589'824) return 50;
return 51;
}
std::string profile_level_id() const {
std::ostringstream output;
output << "42c0" << std::hex << std::setfill('0') << std::setw(2)
<< h264_level();
return output.str();
}
};
constexpr VideoProfile kDefaultVideoProfile{};
class GateFailure final : public std::runtime_error {
public:
GateFailure(std::string stage, std::string detail, HRESULT result = S_OK)
: std::runtime_error(detail), stage_(std::move(stage)), result_(result) {}
const std::string& stage() const noexcept { return stage_; }
HRESULT result() const noexcept { return result_; }
private:
std::string stage_;
HRESULT result_;
};
[[noreturn]] void Fail(const std::string& stage, const std::string& detail);
void Check(HRESULT result, const std::string& stage);
struct Runtime final {
Runtime() {
Check(CoInitializeEx(nullptr, COINIT_MULTITHREADED), "com-initialize");
com_started = true;
Check(MFStartup(MF_VERSION, MFSTARTUP_FULL), "mf-startup");
mf_started = true;
}
~Runtime() {
if (mf_started) {
MFShutdown();
}
if (com_started) {
CoUninitialize();
}
}
bool com_started = false;
bool mf_started = false;
};
struct Adapter final {
UINT index = 0;
DXGI_ADAPTER_DESC1 description = {};
ComPtr<IDXGIAdapter1> value;
};
struct DeviceContext final {
ComPtr<ID3D11Device> device;
ComPtr<ID3D11DeviceContext> context;
ComPtr<IMFDXGIDeviceManager> manager;
D3D_FEATURE_LEVEL feature_level = D3D_FEATURE_LEVEL_9_1;
};
class ActivationList final {
public:
ActivationList() = default;
ActivationList(const ActivationList&) = delete;
ActivationList& operator=(const ActivationList&) = delete;
ActivationList(ActivationList&& other) noexcept
: values(std::exchange(other.values, nullptr)),
count(std::exchange(other.count, 0)) {}
~ActivationList() {
for (UINT32 index = 0; index < count; ++index) {
values[index]->Release();
}
CoTaskMemFree(values);
}
IMFActivate** values = nullptr;
UINT32 count = 0;
};
struct SelectedTransform final {
SelectedTransform() = default;
SelectedTransform(const SelectedTransform&) = delete;
SelectedTransform& operator=(const SelectedTransform&) = delete;
SelectedTransform(SelectedTransform&&) noexcept = default;
SelectedTransform& operator=(SelectedTransform&&) = delete;
~SelectedTransform() { if (activation) activation->ShutdownObject(); }
std::string name;
std::string clsid;
ComPtr<IMFActivate> activation;
ComPtr<IMFTransform> transform;
ComPtr<ICodecAPI> codec;
ComPtr<IMFMediaEventGenerator> events;
};
struct NalSummary final {
bool annex_b = false;
bool sps = false;
bool pps = false;
bool idr = false;
std::optional<std::string> profile_level_id;
};
std::vector<Adapter> EnumerateAdapters();
const Adapter& SelectAdapter(const std::vector<Adapter>& adapters, UINT index);
DeviceContext CreateDevice(const Adapter& adapter);
ActivationList EnumerateHardwareEncoders(const Adapter& adapter,
bool require_candidate = true);
SelectedTransform ActivateTransform(const ActivationList& list, UINT index,
IMFDXGIDeviceManager* manager);
std::wstring GetOptionalString(IMFAttributes* attributes, const GUID& key);
std::string NarrowAscii(const std::wstring& value);
std::string GuidString(const GUID& value);
// The hardware fixture uses the same configuration and output validation.
void SetU32(ICodecAPI* codec, const GUID& key, UINT32 expected,
const std::string& name);
void ConfigureCodec(ICodecAPI* codec,
const VideoProfile& profile = kDefaultVideoProfile);
ComPtr<IMFMediaType> CreateOutputType(
const VideoProfile& profile = kDefaultVideoProfile);
ComPtr<IMFMediaType> CreateInputType(
const VideoProfile& profile = kDefaultVideoProfile);
void ValidateMediaTypes(IMFTransform* transform,
const VideoProfile& profile = kDefaultVideoProfile);
void ValidateCodecReadback(ICodecAPI* codec,
const VideoProfile& profile = kDefaultVideoProfile);
void ForceKeyFrame(ICodecAPI* codec);
ComPtr<IMFSample> PullOutput(IMFTransform* transform,
const MFT_OUTPUT_STREAM_INFO& info);
NalSummary InspectAnnexB(const std::vector<UINT8>& bytes);
std::vector<UINT8> ReadSample(IMFSample* sample);
struct EncodedAccessUnit final {
UINT64 timestamp100ns = 0;
bool key_frame = false;
std::vector<UINT8> bytes;
};
using EncoderClock = std::chrono::steady_clock;
void RequireEncoderTime(EncoderClock::time_point deadline);
class VideoEncoder {
public:
VideoEncoder(OutputKind kind, std::string name, std::string identity)
: kind(kind), name(std::move(name)), identity(std::move(identity)) {}
virtual ~VideoEncoder() = default;
virtual EncodedAccessUnit Encode(ID3D11Texture2D* texture,
UINT64 timestamp100ns,
bool force_key_frame,
EncoderClock::time_point deadline =
EncoderClock::time_point::max()) = 0;
virtual void SetBitrate(UINT32 bitrate) = 0;
const OutputKind kind;
const std::string name;
const std::string identity;
};
// The caller owns COM/MF startup and serializes access to each encoder.
class LiveEncoder final : public VideoEncoder {
public:
explicit LiveEncoder(SelectedTransform selected,
VideoProfile profile = kDefaultVideoProfile);
~LiveEncoder() override;
LiveEncoder(const LiveEncoder&) = delete;
LiveEncoder& operator=(const LiveEncoder&) = delete;
void SetBitrate(UINT32 bitrate) override;
EncodedAccessUnit Encode(
ID3D11Texture2D* texture, UINT64 timestamp100ns, bool force_key_frame,
EncoderClock::time_point deadline = EncoderClock::time_point::max()) override;
private:
MediaEventType NextEvent(EncoderClock::time_point deadline);
void WaitForInput(EncoderClock::time_point deadline);
SelectedTransform selected_;
VideoProfile profile_;
MFT_OUTPUT_STREAM_INFO output_info_ = {};
UINT32 input_requests_ = 0;
std::optional<std::string> profile_level_id_;
bool started_ = false;
};
} // namespace screener::capture::windows
native/capture/windows/libvpx-dependencies.json
-17
View File
@@ -1,17 +0,0 @@
{
"libvpx": {
"version": "1.17.0",
"commit": "6df3ec34557879fff673706f4a1d9fbd0f3a6f0e",
"url": "https://github.com/webmproject/libvpx/archive/6df3ec34557879fff673706f4a1d9fbd0f3a6f0e.tar.gz",
"sha256": "073bd6202e4c55c8cb759abbb123aecd4cc8d99f52ba27af790178cfb1108423"
},
"nasm": {
"version": "2.16.03",
"url": "https://www.nasm.us/pub/nasm/releasebuilds/2.16.03/win64/nasm-2.16.03-win64.zip",
"sha256": "3ee4782247bcb874378d02f7eab4e294a84d3d15f3f6ee2de2f47a46aa7226e6"
},
"make": {
"url": "https://repo.msys2.org/msys/x86_64/make-4.4.1-2-x86_64.pkg.tar.zst",
"sha256": "2408af61717dae87b00c855b132769a125c708907fc94a46bb16dae076113e5c"
}
}
native/capture/windows/main.cpp
+625 -1167
View File
File diff suppressed because it is too large. Load diff
native/capture/windows/output_mailbox.h
+108
View File
@@ -0,0 +1,108 @@
#pragma once
#include <condition_variable>
#include <cstdint>
#include <memory>
#include <mutex>
#include <stdexcept>
#include <utility>
namespace screener::capture {
// One codec worker consumes raw inputs; only its not-yet-encoded input is replaceable.
template <typename Frame>
class OutputMailbox final {
public:
enum class Action { frame, retire, stop };
struct Work final {
Action action = Action::stop;
std::shared_ptr<const Frame> input;
uint64_t generation = 0;
uint32_t bitrate = 0;
bool recovery = false;
};
explicit OutputMailbox(uint32_t ceiling, bool active = true)
: ceiling_(ceiling), bitrate_(ceiling), active_(active) {}
void Submit(std::shared_ptr<const Frame> input) {
std::lock_guard<std::mutex> lock(mutex_);
if (active_ && !stopped_ && !failed_) {
pending_ = std::move(input);
changed_.notify_one();
}
}
Work Take() {
std::unique_lock<std::mutex> lock(mutex_);
changed_.wait(lock, [&]() { return stopped_ || retire_ || pending_; });
if (stopped_) return {};
if (retire_) {
retire_ = false;
return {Action::retire, {}, generation_, bitrate_, false};
}
return {Action::frame, std::exchange(pending_, {}), generation_, bitrate_,
std::exchange(key_requested_, false)};
}
// Only a fresh activation needs a retained capture frame.
bool SetActive(bool active) {
std::lock_guard<std::mutex> lock(mutex_);
if (active_ == active || failed_) return false;
active_ = active;
++generation_;
key_requested_ = true;
if (!active) {
pending_.reset();
retire_ = true;
}
changed_.notify_one();
return active;
}
void RequestKeyFrame() {
std::lock_guard<std::mutex> lock(mutex_);
key_requested_ = true;
}
void SetBitrate(uint32_t bitrate) {
if (bitrate < 1'000 || bitrate > ceiling_) throw std::runtime_error("Output bitrate exceeds its codec bounds");
std::lock_guard<std::mutex> lock(mutex_);
bitrate_ = bitrate;
}
bool Accept(uint64_t generation) {
std::lock_guard<std::mutex> lock(mutex_);
return !stopped_ && !failed_ && active_ && generation == generation_;
}
bool Fail(bool recovery) {
std::lock_guard<std::mutex> lock(mutex_);
key_requested_ = key_requested_ || recovery;
failed_ = true;
pending_.reset();
return !stopped_;
}
void Stop() {
std::lock_guard<std::mutex> lock(mutex_);
stopped_ = true;
pending_.reset();
changed_.notify_one();
}
private:
const uint32_t ceiling_;
std::mutex mutex_;
std::condition_variable changed_;
std::shared_ptr<const Frame> pending_;
uint32_t bitrate_;
uint64_t generation_ = 0;
bool active_ = true;
bool stopped_ = false;
bool failed_ = false;
bool retire_ = false;
bool key_requested_ = true;
};
} // namespace screener::capture
native/capture/windows/vp8_decoder.h
+75
View File
@@ -0,0 +1,75 @@
#pragma once
#include <vpx/vp8dx.h>
#include <vpx/vpx_decoder.h>
#include <algorithm>
#include <cstdint>
#include <stdexcept>
#include <vector>
namespace screener::capture {
// Decoder image storage belongs to libvpx until the next Decode call. A single
// NV12 copy is shared by all derived-output workers after upload.
class Vp8Decoder final {
public:
struct Image final {
uint32_t width = 0;
uint32_t height = 0;
std::vector<uint8_t> nv12;
};
Vp8Decoder() {
vpx_codec_dec_cfg_t config{};
config.threads = 2;
if (vpx_codec_dec_init(&codec_, vpx_codec_vp8_dx(), &config, 0) != VPX_CODEC_OK) {
throw std::runtime_error("VP8 decoder initialization failed");
}
}
~Vp8Decoder() { vpx_codec_destroy(&codec_); }
Vp8Decoder(const Vp8Decoder&) = delete;
Vp8Decoder& operator=(const Vp8Decoder&) = delete;
Image Decode(const std::vector<uint8_t>& data) {
if (data.empty() || data.size() > 4 * 1024 * 1024) throw std::runtime_error("Invalid VP8 decoder input size");
if (!(data[0] & 1)) {
vpx_codec_stream_info_t info{};
info.sz = sizeof(info);
if (vpx_codec_peek_stream_info(vpx_codec_vp8_dx(), data.data(), static_cast<unsigned int>(data.size()), &info) != VPX_CODEC_OK ||
info.w < 2 || info.h < 2 || info.w > 2560 || info.h > 1440 || (info.w & 1) || (info.h & 1)) {
throw std::runtime_error("VP8 decoder input dimensions exceed the source bound");
}
}
if (vpx_codec_decode(&codec_, data.data(), static_cast<unsigned int>(data.size()), nullptr, 0) != VPX_CODEC_OK) {
throw std::runtime_error("VP8 decoder rejected its input");
}
vpx_codec_iter_t iterator = nullptr;
const auto* image = vpx_codec_get_frame(&codec_, &iterator);
if (!image) return {};
const uint32_t width = image->d_w, height = image->d_h;
if (width < 2 || height < 2 || width > 2560 || height > 1440 || (width & 1) || (height & 1) ||
image->fmt != VPX_IMG_FMT_I420 || image->stride[0] < static_cast<int>(width) ||
image->stride[1] < static_cast<int>(width / 2) || image->stride[2] < static_cast<int>(width / 2)) {
throw std::runtime_error("VP8 decoder output format is unsupported");
}
Image result{width, height, std::vector<uint8_t>(static_cast<size_t>(width) * height * 3 / 2)};
for (uint32_t row = 0; row < height; ++row) {
std::copy_n(image->planes[0] + row * image->stride[0], width, result.nv12.data() + row * width);
}
auto* chroma = result.nv12.data() + static_cast<size_t>(width) * height;
for (uint32_t row = 0; row < height / 2; ++row) {
for (uint32_t column = 0; column < width / 2; ++column) {
chroma[row * width + 2 * column] = image->planes[1][row * image->stride[1] + column];
chroma[row * width + 2 * column + 1] = image->planes[2][row * image->stride[2] + column];
}
}
if (vpx_codec_get_frame(&codec_, &iterator)) throw std::runtime_error("VP8 decoder emitted multiple frames for one access unit");
return result;
}
private:
vpx_codec_ctx_t codec_{};
};
} // namespace screener::capture
native/capture/windows/vp8_encoder.cpp
+10 -2
View File
@@ -28,7 +28,7 @@ Vp8Encoder::Vp8Encoder(uint32_t width, uint32_t height, uint32_t frame_rate,
throw std::runtime_error("VP8 encoder profile is invalid");
}
duration100ns_ = 10'000'000 / frame_rate;
vpx_codec_enc_cfg_t config{};
auto& config = config_;
CheckVpx(vpx_codec_enc_config_default(vpx_codec_vp8_cx(), &config, 0));
config.g_w = width;
config.g_h = height;
@@ -57,6 +57,14 @@ Vp8Encoder::~Vp8Encoder() { vpx_codec_destroy(&codec_); }
const char* Vp8Encoder::Version() { return vpx_codec_version_str(); }
void Vp8Encoder::SetBitrate(uint32_t bit_rate) {
if (bit_rate < 1'000) throw std::runtime_error("VP8 bitrate is invalid");
auto updated = config_;
updated.rc_target_bitrate = bit_rate / 1'000;
CheckVpx(vpx_codec_enc_config_set(&codec_, &updated));
config_ = updated;
}
Vp8Frame Vp8Encoder::Encode(uint8_t* nv12, size_t row_pitch,
uint64_t timestamp100ns, bool key_frame) {
if (nv12 == nullptr || row_pitch < width_ ||
@@ -89,7 +97,7 @@ Vp8Frame Vp8Encoder::Encode(uint8_t* nv12, size_t row_pitch,
frame.bytes.assign(bytes, bytes + packet->data.frame.sz);
frame.key_frame = (packet->data.frame.flags & VPX_FRAME_IS_KEY) != 0;
}
if (frame.bytes.empty() || (key_frame && !frame.key_frame)) {
if (!frame.bytes.empty() && key_frame && !frame.key_frame) {
throw std::runtime_error("VP8 encoder did not produce the requested frame");
}
return frame;
native/capture/windows/vp8_encoder.h
+3
View File
@@ -21,12 +21,15 @@ class Vp8Encoder final {
Vp8Encoder(const Vp8Encoder&) = delete;
Vp8Encoder& operator=(const Vp8Encoder&) = delete;
// An empty result is a codec-owned frame drop, not an encoder failure.
Vp8Frame Encode(uint8_t* nv12, size_t row_pitch, uint64_t timestamp100ns,
bool key_frame);
void SetBitrate(uint32_t bit_rate);
static const char* Version();
private:
vpx_codec_ctx_t codec_{};
vpx_codec_enc_cfg_t config_{};
uint32_t width_;
uint32_t height_;
unsigned long duration100ns_;
native/capture/windows/webrtc-dependencies.json
+19
View File
@@ -0,0 +1,19 @@
{
"webrtc": {
"version": "m152.7977.0.2",
"commit": "6f37672d358475cd17544121a12494da454d85fb",
"url": "https://github.com/shiguredo-webrtc-build/webrtc-build/releases/download/m152.7977.0.2/webrtc.windows_x86_64.zip",
"sha256": "3250ee091eb745b9885482d92db352e6bcd5426940021ff90a095747d84d2137"
},
"msvc": {
"directory": "Contents/VC/Tools/MSVC/14.44.35207",
"crt": {
"url": "https://download.visualstudio.microsoft.com/download/pr/67cf767c-5e71-47c2-a54a-cd5631e28942/f01f701a7bcd9587a340898c851424f6a52bb913a70c185ff0d5bf0288c5831a/Microsoft.VC.14.44.17.14.CRT.x64.Desktop.base.vsix",
"sha256": "f01f701a7bcd9587a340898c851424f6a52bb913a70c185ff0d5bf0288c5831a"
},
"tools": {
"url": "https://download.visualstudio.microsoft.com/download/pr/bbc72d8e-2acd-4229-8f6a-85e23c5e3456/ee0baaa3a112d255f19f6c27dcc0ff6e496949eb9f1f37be0ac908c562a7076c/Microsoft.VC.14.44.17.14.Tools.HostX64.TargetX64.base.vsix",
"sha256": "ee0baaa3a112d255f19f6c27dcc0ff6e496949eb9f1f37be0ac908c562a7076c"
}
}
}
native/fixtures/mf-h264-hardware/build.ps1
+6 -2
View File
@@ -28,15 +28,19 @@ if ([string]::IsNullOrWhiteSpace($installationPath)) {
}
$developerCommand = Join-Path $installationPath 'Common7\Tools\VsDevCmd.bat'
$sourcePath = Join-Path $fixtureDirectory '..\..\client\platform\windows\capture\main.cpp'
$sourcePath = Join-Path $fixtureDirectory '..\..\capture\windows\main.cpp'
$encoderSourcePath = Join-Path $fixtureDirectory '..\..\capture\windows\h264_encoder.cpp'
$executablePath = Join-Path $outputPath 'screener-mf-h264-fixture.exe'
$objectPath = Join-Path $outputPath 'main.obj'
$encoderObjectPath = Join-Path $outputPath 'h264-encoder.obj'
New-Item -ItemType Directory -Path $outputPath -Force | Out-Null
$compile = @(
'call "{0}" -arch=x64 -host_arch=x64 >nul' -f $developerCommand
'cl.exe /nologo /std:c++20 /EHsc /W4 /WX /DSCREENER_H264_FIXTURE /DUNICODE /D_UNICODE /DWIN32_LEAN_AND_MEAN /D_WIN32_WINNT=0x0A00 "{0}" /Fo:"{1}" /Fe:"{2}" /link mfplat.lib mf.lib mfuuid.lib d3d11.lib dxgi.lib dxguid.lib evr.lib ole32.lib oleaut32.lib pdh.lib windowsapp.lib' -f $sourcePath, $objectPath, $executablePath
'cl.exe /nologo /c /std:c++20 /EHsc /W4 /WX /DSCREENER_H264_FIXTURE /DUNICODE /D_UNICODE /DWIN32_LEAN_AND_MEAN /D_WIN32_WINNT=0x0A00 "{0}" /Fo:"{1}"' -f $sourcePath, $objectPath
'cl.exe /nologo /c /std:c++20 /EHsc /W4 /WX /DUNICODE /D_UNICODE /DWIN32_LEAN_AND_MEAN /D_WIN32_WINNT=0x0A00 "{0}" /Fo:"{1}"' -f $encoderSourcePath, $encoderObjectPath
'link.exe /nologo /out:"{0}" "{1}" "{2}" mfplat.lib mf.lib mfuuid.lib d3d11.lib dxgi.lib dxguid.lib evr.lib ole32.lib oleaut32.lib pdh.lib windowsapp.lib' -f $executablePath, $objectPath, $encoderObjectPath
) -join ' && '
& cmd.exe /d /s /c $compile
native/probes/encoder-pool/CMakeLists.txt
+27
View File
@@ -0,0 +1,27 @@
cmake_minimum_required(VERSION 3.20)
project(screener_encoder_pool_probe LANGUAGES CXX)
set(CMAKE_CXX_STANDARD 20)
set(CMAKE_MSVC_RUNTIME_LIBRARY MultiThreaded)
set(WEBRTC_SDK "" CACHE PATH "Extracted matching Shiguredo WebRTC SDK")
set(MSVC_CRT_DIR "" CACHE PATH "Optional newer official MSVC x64 library directory")
if(NOT EXISTS "${WEBRTC_SDK}/lib/webrtc.lib")
message(FATAL_ERROR "This Windows probe needs WEBRTC_SDK pointing to the matching SDK")
endif()
add_executable(encoder-pool-probe harness.cpp h264_encoder_adapter.cpp
../../capture/windows/h264_encoder.cpp)
target_include_directories(encoder-pool-probe SYSTEM PRIVATE
"${WEBRTC_SDK}/include"
"${WEBRTC_SDK}/include/third_party/abseil-cpp"
"${WEBRTC_SDK}/include/third_party/boringssl/src/include"
"${WEBRTC_SDK}/include/third_party/libyuv/include"
"${WEBRTC_SDK}/include/third_party/libvpx/source/libvpx")
target_compile_definitions(encoder-pool-probe PRIVATE
NDEBUG _ITERATOR_DEBUG_LEVEL=0 WEBRTC_WIN RTC_ENABLE_H265 NOMINMAX WIN32_LEAN_AND_MEAN _WIN32_WINNT=0x0A00)
target_compile_options(encoder-pool-probe PRIVATE /EHsc /GR /O2 /W3)
if(MSVC_CRT_DIR)
target_link_directories(encoder-pool-probe BEFORE PRIVATE "${MSVC_CRT_DIR}")
endif()
target_link_libraries(encoder-pool-probe PRIVATE
"${WEBRTC_SDK}/lib/webrtc.lib"
winmm ws2_32 Strmiids crypt32 dmoguids iphlpapi msdmo Secur32 wmcodecdspuuid
ole32 oleaut32 mfplat mf mfuuid d3d11 dxgi dxguid evr)
native/probes/encoder-pool/README.md
+99
View File
@@ -0,0 +1,99 @@
# Encoder Pool Feasibility Probe
Explicit Windows experiment, not a product dependency or CI job. Two real
libwebrtc VideoStreamEncoders consume one synthetic 640x360 source through
independent stock VideoAdapters. It does not capture the desktop or open ports.
[Research and evidence](../../../docs/research/webrtc-encoder-pool.md) own conclusions.
Use Shiguredo `m152.7977.0.2` Windows x64 SDK (WebRTC commit
`6f37672d358475cd17544121a12494da454d85fb`), matching its headers and static library:
[download](https://github.com/shiguredo-webrtc-build/webrtc-build/releases/download/m152.7977.0.2/webrtc.windows_x86_64.zip).
ZIP SHA-256: `3250ee091eb745b9885482d92db352e6bcd5426940021ff90a095747d84d2137`.
Keep the extracted SDK, official compiler dependencies and executable under
ignored `build/encoder-pool`; preserve the SDK's notices. Do not ship this SDK
as part of Screener based on this probe.
With CMake and Visual Studio 2022 x64 tools available:
```powershell
cmake -S native/probes/encoder-pool -B build/encoder-pool/probe -A x64 `
-DWEBRTC_SDK="$PWD/build/encoder-pool/webrtc"
cmake --build build/encoder-pool/probe --config Release -j 2
& ./build/encoder-pool/probe/Release/encoder-pool-probe.exe
& ./build/encoder-pool/probe/Release/encoder-pool-probe.exe --pooled
& ./build/encoder-pool/probe/Release/encoder-pool-probe.exe --pooled --group-adjuster
```
The tested older compiler was MSVC 14.42; linking requires the newer official
14.44 libraries/linker used by the SDK. On such a machine pass `MSVC_CRT_DIR`
to CMake and `/p:LinkToolPath=<official tools directory>` to MSBuild after `--`.
Do not supply substitute STL implementations or alter global tool settings.
The full trace takes about 51 seconds: healthy, 100 kbps weak B, release to
2 Mbps, one omitted B input, then independent B retirement. `--healthy-only`
omits the weak/release phases. JSONL reports real encoding calls, both WebRTC
decoders, raw libvpx hashes without history-dependent postprocessing, dimensions,
first transition times, group lifetime and resource observations. `integrity`
checks nonzero decoding, error counts, shared-payload comparisons and retirement;
it is not an automatic performance/parity verdict.
`--no-adjuster` is a diagnostic ablation only. `--group-adjuster` instead keeps
WebRTC's actual adjustment algorithm once per physical codec; it must not run
alongside per-VSE correction. `--unsafe-skip` removes dependency protection as a
negative control and should fail integrity with group adjustment and a skipped
input. This cache arm is guarded single-source realtime VP8/L1T1 with synchronous
codecs; no H264 cache, physical overload or public-network claim follows.
`--realtime` removes per-input encoder-queue drains. `--latency-probe` requires
that mode and inserts a bounded pixel-proportional 45 ms codec delay, then
releases it; it is not a CPU/GPU stress test. `--shared-pipeline --realtime
--latency-probe` is a separate control with one full stock VSE feeding both
decoders, not two VSEs borrowing its encoder. Do not combine it with pool or
adjuster flags. Source and codec work remain bounded; use a 95-second process
deadline for these approximately 70-second traces. The shared-pipeline B resource
metrics repeat the common pipeline, while its decoded counters are independent.
## Hardware H264 Pipeline
The separate `--h264` arm wraps the existing Windows MFT `LiveEncoder` in a
WebRTC encoder adapter. It requires `--shared-pipeline --healthy-only --export
PATH`; one complete stock VSE supplies two output callbacks. It does not test
two independent controllers borrowing an H264 encoder. Run it alone, with a
30-second process deadline and no concurrent capture/codec workload:
```powershell
& ./build/encoder-pool/probe/Release/encoder-pool-probe.exe `
--h264 --shared-pipeline --healthy-only --realtime `
--export build/encoder-pool/h264-input.jsonl
```
This is synthetic I420 upload to NV12/MFT, not WGC capture or zero-copy input.
The pinned SDK has no H264 decoder, so this arm records payload equality and
encoded dimensions with decoded counters at zero. Validate the exported AUs
separately in Chrome. The existing `scripts/encoded-group-decode.mjs` accepts
`SCREENER_ENCODED_CODEC=avc1.42c033` and `SCREENER_ENCODED_OUTPUT` pointing to two
arrays of `{Index, Width, Height, PTS, Recovery, Data}` frames (nanosecond PTS,
base64 Annex-B data). A repeated export in those two arrays proves decoding of
that bitstream, not two Pion transports or weak-budget/resource adaptation.
## Healthy Relay Chain
Export the first 180 real AUs from sender A for the opt-in Go forwarding check:
```powershell
& ./build/encoder-pool/probe/Release/encoder-pool-probe.exe `
--pooled --group-adjuster --healthy-only --export build/encoder-pool/chain-input.jsonl
go test -c -o build/embedded-media/mediaedge.test.exe ./internal/client/mediaedge
$env:SCREENER_POOL_CHAIN_FIXTURE = "$PWD/build/encoder-pool/chain-input.jsonl"
$env:SCREENER_NATIVE_CAPTURE = "$PWD/build/client-check/screener-client-capture.exe"
$env:SCREENER_POOL_CHAIN_OUTPUT = "$PWD/build/encoder-pool/chain-result.json"
& ./build/embedded-media/mediaedge.test.exe `
'-test.run=^TestRelayChainEncodedFixture$' '-test.v' '-test.timeout=30s'
```
The real capture executable must already be built, so Native relay capability
remains enabled while checking that healthy forwarding does not start it. This
replays encoded video through two Native receivers and three leaves, checking
recovery-first, contiguous, hash-identical AUs and closure. No frame payload is
committed. Optional output contains only the synthetic trace's summary. It is
not a live encoder/BWE integration or an internet/NAT test.
native/probes/encoder-pool/h264_encoder_adapter.cpp
+310
View File
@@ -0,0 +1,310 @@
#include "h264_encoder_adapter.h"
#include "../../capture/windows/h264_encoder.h"
#include <algorithm>
#include <cmath>
#include <iostream>
#include <limits>
#include <utility>
#include <vector>
#include "api/video_codecs/h264_profile_level_id.h"
#include "libyuv/convert_from.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "modules/video_coding/include/video_error_codes.h"
#include "system_wrappers/include/clock.h"
namespace screener::probe {
namespace {
namespace native = screener::capture::windows;
bool ValidDimensions(int width, int height) {
return width >= 2 && width <= 2560 && height >= 2 && height <= 1440 &&
width % 2 == 0 && height % 2 == 0;
}
webrtc::SdpVideoFormat HardwareFormat() {
return {"H264", {{"profile-level-id", "42c033"},
{"level-asymmetry-allowed", "1"},
{"packetization-mode", "1"}},
{webrtc::ScalabilityMode::kL1T1}};
}
class HardwareEncoder final : public webrtc::VideoEncoder {
public:
HardwareEncoder(const webrtc::Environment& env, unsigned adapter_index,
unsigned encoder_index, webrtc::H264Level maximum_level)
: env_(env), adapter_index_(adapter_index), encoder_index_(encoder_index),
maximum_level_(maximum_level) {}
~HardwareEncoder() override { Release(); }
int InitEncode(const webrtc::VideoCodec* config, const Settings&) override {
Release();
if (!config || config->codecType != webrtc::kVideoCodecH264 ||
!ValidDimensions(config->width, config->height) ||
config->maxFramerate == 0 || config->maxFramerate > 60 ||
config->startBitrate == 0 ||
config->startBitrate > std::numeric_limits<UINT32>::max() / 1000) {
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
if (config->numberOfSimulcastStreams > 1 ||
config->H264().numberOfTemporalLayers > 1 ||
(config->GetScalabilityMode() &&
config->GetScalabilityMode() != webrtc::ScalabilityMode::kL1T1)) {
return WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED;
}
profile_.width = config->width;
profile_.height = config->height;
profile_.frame_rate = config->maxFramerate;
profile_.bit_rate = config->startBitrate * 1000;
if (profile_.h264_level() > static_cast<UINT32>(maximum_level_)) {
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
try {
runtime_ = std::make_unique<native::Runtime>();
auto adapters = native::EnumerateAdapters();
const auto& adapter = native::SelectAdapter(adapters, adapter_index_);
device_ = native::CreateDevice(adapter);
auto activations = native::EnumerateHardwareEncoders(adapter);
encoder_ = std::make_unique<native::LiveEncoder>(
native::ActivateTransform(activations, encoder_index_, device_.manager.Get()),
profile_);
nv12_.resize(static_cast<size_t>(profile_.width) * profile_.height * 3 / 2);
implementation_name_ = encoder_->name + " (" + encoder_->identity + ")";
applied_bitrate_ = profile_.bit_rate;
content_type_ = config->mode == webrtc::VideoCodecMode::kScreensharing
? webrtc::VideoContentType::SCREENSHARE
: webrtc::VideoContentType::UNSPECIFIED;
return WEBRTC_VIDEO_CODEC_OK;
} catch (const std::exception& error) {
ReportFailure("initialize", error);
Release();
return WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
}
}
int32_t RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback* callback) override {
callback_ = callback;
return WEBRTC_VIDEO_CODEC_OK;
}
int32_t Release() override {
encoder_.reset();
device_ = {};
runtime_.reset();
nv12_.clear();
applied_bitrate_ = 0;
paused_ = false;
key_frame_pending_ = true;
error_ = WEBRTC_VIDEO_CODEC_OK;
return WEBRTC_VIDEO_CODEC_OK;
}
void SetRates(const RateControlParameters& rates) override {
if (!encoder_ || error_ != WEBRTC_VIDEO_CODEC_OK) return;
const UINT32 bitrate = rates.bitrate.get_sum_bps();
if (bitrate != rates.bitrate.GetBitrate(0, 0) ||
!std::isfinite(rates.framerate_fps)) {
error_ = WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
return;
}
if (bitrate == 0) {
paused_ = true;
return;
}
const double fps = rates.framerate_fps > 0
? rates.framerate_fps : profile_.frame_rate;
// Chromium M152 MF keeps the configured FPS and compensates its bitrate:
// media/gpu/windows/media_foundation_video_encode_accelerator_win.cc,
// AdjustBitrateToFrameRate (152.0.7977.82).
const double adjusted = static_cast<double>(bitrate) * profile_.frame_rate / fps;
if (!std::isfinite(adjusted) || adjusted < 1 ||
adjusted > std::numeric_limits<UINT32>::max()) {
error_ = WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
return;
}
const auto next_bitrate = static_cast<UINT32>(adjusted);
try {
if (next_bitrate != applied_bitrate_) encoder_->SetBitrate(next_bitrate);
applied_bitrate_ = next_bitrate;
key_frame_pending_ = key_frame_pending_ || paused_;
paused_ = false;
} catch (const std::exception& error) {
ReportFailure("rates", error);
error_ = WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
}
}
int32_t Encode(const webrtc::VideoFrame& frame,
const std::vector<webrtc::VideoFrameType>* requested) override {
if (!encoder_ || !callback_) return WEBRTC_VIDEO_CODEC_UNINITIALIZED;
if (error_ != WEBRTC_VIDEO_CODEC_OK) return error_;
key_frame_pending_ = key_frame_pending_ ||
(requested && std::find(requested->begin(), requested->end(),
webrtc::VideoFrameType::kVideoFrameKey) != requested->end());
if (paused_ || (requested && !requested->empty() &&
requested->front() == webrtc::VideoFrameType::kEmptyFrame)) {
callback_->OnFrameDropped(frame.rtp_timestamp(), 0, true);
return WEBRTC_VIDEO_CODEC_OK;
}
if (!frame.video_frame_buffer() || frame.width() != static_cast<int>(profile_.width) ||
frame.height() != static_cast<int>(profile_.height) || frame.timestamp_us() < 0 ||
frame.timestamp_us() > std::numeric_limits<LONGLONG>::max() / 10) {
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
try {
const auto began = env_.clock().TimeInMilliseconds();
auto input = frame.video_frame_buffer()->ToI420();
if (!input || libyuv::I420ToNV12(
input->DataY(), input->StrideY(), input->DataU(), input->StrideU(),
input->DataV(), input->StrideV(), nv12_.data(), profile_.width,
nv12_.data() + static_cast<size_t>(profile_.width) * profile_.height,
profile_.width, profile_.width, profile_.height) != 0) {
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
}
D3D11_TEXTURE2D_DESC description{};
description.Width = profile_.width;
description.Height = profile_.height;
description.MipLevels = 1;
description.ArraySize = 1;
description.Format = DXGI_FORMAT_NV12;
description.SampleDesc.Count = 1;
description.Usage = D3D11_USAGE_DEFAULT;
D3D11_SUBRESOURCE_DATA pixels{};
pixels.pSysMem = nv12_.data();
pixels.SysMemPitch = profile_.width;
pixels.SysMemSlicePitch = static_cast<UINT>(nv12_.size());
native::ComPtr<ID3D11Texture2D> texture;
native::Check(device_.device->CreateTexture2D(&description, &pixels, &texture),
"webrtc-h264-upload");
const auto access_unit = encoder_->Encode(
texture.Get(), static_cast<UINT64>(frame.timestamp_us()) * 10,
key_frame_pending_);
key_frame_pending_ = false;
webrtc::EncodedImage image;
image.SetEncodedData(webrtc::EncodedImageBuffer::Create(
access_unit.bytes.data(), access_unit.bytes.size()));
image._encodedWidth = profile_.width;
image._encodedHeight = profile_.height;
image.SetRtpTimestamp(frame.rtp_timestamp());
image.capture_time_ms_ = frame.timestamp_us() / 1000;
image.ntp_time_ms_ = frame.ntp_time_ms();
image.SetPresentationTimestamp(frame.presentation_timestamp());
image.rotation_ = frame.rotation();
image.content_type_ = content_type_;
image.SetColorSpace(frame.color_space());
image.SetPacketInfos(frame.packet_infos());
image.SetVideoFrameTrackingId(frame.id());
image.SetSimulcastIndex(0);
image.SetSpatialIndex(0);
image.SetTemporalIndex(0);
image.set_end_of_temporal_unit(true);
image.SetFrameType(access_unit.key_frame
? webrtc::VideoFrameType::kVideoFrameKey
: webrtc::VideoFrameType::kVideoFrameDelta);
image.SetEncodeTime(began, env_.clock().TimeInMilliseconds());
webrtc::CodecSpecificInfo info;
info.codecType = webrtc::kVideoCodecH264;
info.codecSpecific.H264 = {};
info.codecSpecific.H264.packetization_mode = webrtc::H264PacketizationMode::NonInterleaved;
info.codecSpecific.H264.idr_frame = access_unit.key_frame;
info.scalability_mode = webrtc::ScalabilityMode::kL1T1;
const auto result = callback_->OnEncodedImage(image, &info);
return result.error == webrtc::EncodedImageCallback::Result::OK
? WEBRTC_VIDEO_CODEC_OK : WEBRTC_VIDEO_CODEC_ERROR;
} catch (const std::exception& error) {
ReportFailure("encode", error);
error_ = WEBRTC_VIDEO_CODEC_ENCODER_FAILURE;
return error_;
}
}
EncoderInfo GetEncoderInfo() const override {
EncoderInfo info;
info.implementation_name = implementation_name_;
info.is_hardware_accelerated = true;
info.enable_cpu_overuse_detection = true;
info.has_trusted_rate_controller = false;
info.is_qp_trusted = false;
info.scaling_settings = ScalingSettings::kOff;
info.requested_resolution_alignment = 2;
info.supports_native_handle = false;
info.supports_simulcast = false;
info.preferred_pixel_formats = {webrtc::VideoFrameBuffer::Type::kI420};
return info;
}
private:
static void ReportFailure(const char* operation, const std::exception& error) {
std::cerr << "hardware-h264 " << operation << ": " << error.what();
if (const auto* failure = dynamic_cast<const native::GateFailure*>(&error)) {
std::cerr << " stage=" << failure->stage() << " hresult=" << failure->result();
}
std::cerr << '\n';
}
const webrtc::Environment env_;
const unsigned adapter_index_;
const unsigned encoder_index_;
const webrtc::H264Level maximum_level_;
std::unique_ptr<native::Runtime> runtime_;
native::DeviceContext device_;
std::unique_ptr<native::LiveEncoder> encoder_;
native::VideoProfile profile_;
std::vector<UINT8> nv12_;
webrtc::EncodedImageCallback* callback_ = nullptr;
std::string implementation_name_ = "Windows Media Foundation H264";
webrtc::VideoContentType content_type_ = webrtc::VideoContentType::UNSPECIFIED;
UINT32 applied_bitrate_ = 0;
int32_t error_ = WEBRTC_VIDEO_CODEC_OK;
bool paused_ = false;
bool key_frame_pending_ = true;
};
class HardwareFactory final : public webrtc::VideoEncoderFactory {
public:
HardwareFactory(unsigned adapter_index, unsigned encoder_index)
: adapter_index_(adapter_index), encoder_index_(encoder_index) {}
std::vector<webrtc::SdpVideoFormat> GetSupportedFormats() const override {
return {HardwareFormat()};
}
CodecSupport QueryCodecSupport(const webrtc::SdpVideoFormat& format,
std::optional<std::string> scalability_mode,
std::optional<webrtc::Resolution> resolution) const override {
if (resolution && !ValidDimensions(resolution->width, resolution->height)) return {};
const auto profile = webrtc::ParseSdpForH264ProfileLevelId(format.parameters);
native::VideoProfile minimum;
minimum.frame_rate = 1;
if (resolution) {
minimum.width = resolution->width;
minimum.height = resolution->height;
}
const bool supported = format.IsSameCodec(HardwareFormat()) &&
(!scalability_mode || *scalability_mode == "L1T1") && profile &&
minimum.h264_level() <= static_cast<UINT32>(profile->level);
return {supported, supported};
}
std::unique_ptr<webrtc::VideoEncoder> Create(
const webrtc::Environment& env, const webrtc::SdpVideoFormat& format) override {
if (!QueryCodecSupport(format, std::nullopt, std::nullopt).is_supported) return nullptr;
const auto profile = webrtc::ParseSdpForH264ProfileLevelId(format.parameters);
return std::make_unique<HardwareEncoder>(env, adapter_index_, encoder_index_, profile->level);
}
private:
const unsigned adapter_index_;
const unsigned encoder_index_;
};
} // namespace
std::unique_ptr<webrtc::VideoEncoderFactory> CreateHardwareEncoderFactory(
unsigned adapter_index, unsigned encoder_index) {
return std::make_unique<HardwareFactory>(adapter_index, encoder_index);
}
} // namespace screener::probe
native/probes/encoder-pool/h264_encoder_adapter.h
+13
View File
@@ -0,0 +1,13 @@
#pragma once
#include <memory>
#include "api/video_codecs/video_encoder_factory.h"
namespace screener::probe {
// CPU I420 input only. InitEncode and Release own COM/MF on the encoder queue.
std::unique_ptr<webrtc::VideoEncoderFactory> CreateHardwareEncoderFactory(
unsigned adapter_index = 0, unsigned encoder_index = 0);
} // namespace screener::probe
native/probes/encoder-pool/harness.cpp
+567
View File
@@ -0,0 +1,567 @@
// Explicit synthetic experiment. Not part of Client, Server, unit tests or CI.
#include "pool.h"
#include "h264_encoder_adapter.h"
#include <algorithm>
#include <array>
#include <chrono>
#include <cstdint>
#include <exception>
#include <fstream>
#include <future>
#include <iostream>
#include <map>
#include <memory>
#include <mutex>
#include <stdexcept>
#include <string>
#include <thread>
#include "api/environment/environment_factory.h"
#include "api/field_trials_view.h"
#include "api/make_ref_counted.h"
#include "api/task_queue/task_queue_base.h"
#include "api/task_queue/task_queue_factory.h"
#include "api/video/builtin_video_bitrate_allocator_factory.h"
#include "api/video/i420_buffer.h"
#include "api/video/video_adapter.h"
#include "api/video/video_frame.h"
#include "api/video/video_stream_encoder_settings.h"
#include "api/video_codecs/builtin_video_decoder_factory.h"
#include "api/video_codecs/video_decoder.h"
#include "api/video_codecs/video_decoder_factory.h"
#include "call/video_send_stream.h"
#include "system_wrappers/include/clock.h"
#include "video/adaptation/overuse_frame_detector.h"
#include "video/frame_cadence_adapter.h"
#include "video/send_statistics_proxy.h"
#include "video/video_stream_encoder.h"
#include "vpx/vpx_decoder.h"
#include "vpx/vp8dx.h"
namespace {
using namespace webrtc;
constexpr int kWidth = 640;
constexpr int kHeight = 360;
constexpr int kFps = 30;
class Trials final : public FieldTrialsView {
public:
explicit Trials(bool disable) : disable_(disable) {}
std::string Lookup(absl::string_view key) const override {
return disable_ && key == "WebRTC-VideoRateControl" ? "bitrate_adjuster:false" : "";
}
private:
bool disable_;
};
template <typename F>
void Sync(TaskQueueBase* queue, F fn) {
auto completion = std::make_shared<std::promise<void>>();
auto future = completion->get_future();
queue->PostTask([completion, fn = std::move(fn)]() mutable {
try {
fn();
completion->set_value();
} catch (...) {
completion->set_exception(std::current_exception());
}
});
if (future.wait_for(std::chrono::seconds(10)) != std::future_status::ready)
throw std::runtime_error("task queue deadline exceeded");
future.get();
}
uint64_t HashBytes(uint64_t hash, const uint8_t* bytes, size_t count) {
for (size_t i = 0; i < count; ++i) hash = (hash ^ bytes[i]) * 1099511628211ULL;
return hash;
}
struct FrameProof {
uint64_t payload = 0;
uint64_t pixels = 0;
int width = 0;
int height = 0;
};
class SharedProof {
public:
void Observe(int consumer, uint32_t timestamp, FrameProof proof) {
std::lock_guard<std::mutex> lock(mutex_);
auto& row = pending_[timestamp];
row[consumer] = proof;
if (row[0].payload && row[1].payload) {
if (row[0].payload == row[1].payload) {
++compared;
if (row[0].pixels && row[1].pixels) {
++decodedCompared;
if (row[0].pixels != row[1].pixels || row[0].width != row[1].width ||
row[0].height != row[1].height) ++mismatches;
}
}
pending_.erase(timestamp);
}
while (pending_.size() > 64) pending_.erase(pending_.begin());
}
std::atomic<int> compared{0};
std::atomic<int> decodedCompared{0};
std::atomic<int> mismatches{0};
private:
std::mutex mutex_;
std::map<uint32_t, std::array<FrameProof, 2>> pending_;
};
struct Counts {
int encoded = 0;
int decoded = 0;
int errors = 0;
int keyframes = 0;
int width = 0;
int height = 0;
int min_width = kWidth;
int min_height = kHeight;
uint64_t bytes = 0;
double first_full_ms = -1;
double first_reduced_ms = -1;
};
class Sink final : public VideoStreamEncoderInterface::EncoderSink,
public DecodedImageCallback {
public:
Sink(const Environment& env, int consumer, SharedProof& proof, bool h264 = false)
: consumer_(consumer), proof_(proof) {
if (h264) return; // The pinned SDK has no H264 decoder; export to Browser WebCodecs.
auto factory = CreateBuiltinVideoDecoderFactory();
decoder_ = factory->Create(env, SdpVideoFormat("VP8"));
VideoDecoder::Settings settings;
settings.set_codec_type(kVideoCodecVP8);
settings.set_number_of_cores(1);
if (!decoder_ || !decoder_->Configure(settings) ||
decoder_->RegisterDecodeCompleteCallback(this) != 0)
throw std::runtime_error("VP8 decoder initialization failed");
vpx_codec_dec_cfg_t raw_config{1, 0, 0};
if (vpx_codec_dec_init(&raw_, vpx_codec_vp8_dx(), &raw_config, 0) != VPX_CODEC_OK)
throw std::runtime_error("raw VP8 decoder initialization failed");
}
~Sink() override {
if (decoder_) { decoder_->Release(); vpx_codec_destroy(&raw_); }
}
void OnEncoderConfigurationChanged(std::vector<VideoStream>, bool,
VideoEncoderConfig::ContentType, int) override {}
void OnBitrateAllocationUpdated(const VideoBitrateAllocation&) override {}
void OnVideoLayersAllocationUpdated(VideoLayersAllocation) override {}
void OnFrameDropped(uint32_t, int, bool) override {}
Result OnEncodedImage(const EncodedImage& image,
const CodecSpecificInfo*) override {
std::lock_guard<std::mutex> lock(mutex_);
++counts_.encoded;
counts_.bytes += image.size();
counts_.keyframes += image.IsKey();
if (export_.is_open() && counts_.encoded <= 180) {
constexpr char hex[] = "0123456789abcdef";
std::string data;
data.reserve(image.size() * 2);
for (uint8_t byte : image) {
data.push_back(hex[byte >> 4]);
data.push_back(hex[byte & 15]);
}
export_ << "{\"index\":" << counts_.encoded - 1
<< ",\"width\":" << image._encodedWidth
<< ",\"height\":" << image._encodedHeight
<< ",\"recovery\":" << (image.IsKey() ? "true" : "false")
<< ",\"dataHex\":\"" << data << "\"}\n";
if (!export_) ++counts_.errors;
}
current_timestamp_ = image.RtpTimestamp();
current_payload_ = HashBytes(1469598103934665603ULL, image.data(), image.size());
if (!decoder_) {
RecordSize(image._encodedWidth, image._encodedHeight);
proof_.Observe(consumer_, current_timestamp_,
{current_payload_, 0, static_cast<int>(image._encodedWidth), static_cast<int>(image._encodedHeight)});
} else if (decoder_->Decode(image, 0) < 0) ++counts_.errors;
if (decoder_ && vpx_codec_decode(&raw_, image.data(), static_cast<unsigned>(image.size()), nullptr, 0) != VPX_CODEC_OK) {
++counts_.errors;
} else if (decoder_) {
vpx_codec_iter_t iterator = nullptr;
if (const auto* raw = vpx_codec_get_frame(&raw_, &iterator)) {
uint64_t hash = 1469598103934665603ULL;
for (int plane = 0; plane < 3; ++plane) {
const unsigned width = plane ? (raw->d_w + 1) / 2 : raw->d_w;
const unsigned height = plane ? (raw->d_h + 1) / 2 : raw->d_h;
for (unsigned y = 0; y < height; ++y)
hash = HashBytes(hash, raw->planes[plane] + y * raw->stride[plane], width);
}
proof_.Observe(consumer_, current_timestamp_,
{current_payload_, hash, static_cast<int>(raw->d_w), static_cast<int>(raw->d_h)});
} else ++counts_.errors;
}
if (duplicate_) duplicate_->OnEncodedImage(image, nullptr);
return Result(Result::OK);
}
// The selected built-in VP8 decoder completes synchronously inside Decode.
int32_t Decoded(VideoFrame& frame) override {
++counts_.decoded;
RecordSize(frame.width(), frame.height());
return 0;
}
void RecordSize(int width, int height) {
counts_.width = width;
counts_.height = height;
counts_.min_width = std::min(counts_.min_width, width);
counts_.min_height = std::min(counts_.min_height, height);
const double elapsed = std::chrono::duration<double, std::milli>(
std::chrono::steady_clock::now() - phase_started_).count();
if (width == kWidth && height == kHeight && counts_.first_full_ms < 0)
counts_.first_full_ms = elapsed;
if (width < kWidth && counts_.first_reduced_ms < 0)
counts_.first_reduced_ms = elapsed;
}
Counts Snapshot() {
std::lock_guard<std::mutex> lock(mutex_);
return counts_;
}
void Export(const std::string& path) {
export_.open(path, std::ios::binary | std::ios::trunc);
if (!export_) throw std::runtime_error("fixture export unavailable");
}
void SetDuplicate(Sink* sink) {
std::lock_guard<std::mutex> lock(mutex_);
duplicate_ = sink;
}
void FinishExport() {
if (!export_.is_open()) return;
export_.flush();
export_.close();
if (!export_ || counts_.encoded < 180)
throw std::runtime_error("fixture export incomplete");
}
void ResetMinimum() {
std::lock_guard<std::mutex> lock(mutex_);
counts_.min_width = kWidth;
counts_.min_height = kHeight;
counts_.first_full_ms = -1;
counts_.first_reduced_ms = -1;
phase_started_ = std::chrono::steady_clock::now();
}
private:
const int consumer_;
SharedProof& proof_;
std::mutex mutex_;
std::unique_ptr<VideoDecoder> decoder_;
vpx_codec_ctx_t raw_{};
std::ofstream export_;
Sink* duplicate_ = nullptr;
Counts counts_;
uint32_t current_timestamp_ = 0;
uint64_t current_payload_ = 0;
std::chrono::steady_clock::time_point phase_started_ = std::chrono::steady_clock::now();
};
// Each sender applies stock VideoAdapter wants to the same original input.
class Source final : public VideoSourceInterface<VideoFrame> {
public:
void AddOrUpdateSink(VideoSinkInterface<VideoFrame>* sink,
const VideoSinkWants& wants) override {
sink_ = sink;
adapter_.OnSinkWants(wants);
++wants_updates;
min_pixels = std::min(min_pixels, wants.max_pixel_count);
}
void RemoveSink(VideoSinkInterface<VideoFrame>* sink) override {
if (sink_ == sink) sink_ = nullptr;
}
void Push(const VideoFrame& original) {
if (!sink_) return;
int crop_width, crop_height, width, height;
if (!adapter_.AdaptFrameResolution(original.width(), original.height(),
original.timestamp_us() * 1000,
&crop_width, &crop_height, &width, &height)) {
++dropped;
sink_->OnDiscardedFrame();
return;
}
VideoFrame frame = original;
if (width != original.width() || height != original.height() ||
crop_width != original.width() || crop_height != original.height()) {
auto scaled = I420Buffer::Create(width, height);
scaled->CropAndScaleFrom(*original.video_frame_buffer()->ToI420(),
(original.width() - crop_width) / 2,
(original.height() - crop_height) / 2,
crop_width, crop_height);
frame.set_video_frame_buffer(scaled);
}
sink_->OnFrame(frame);
}
int dropped = 0;
int wants_updates = 0;
int min_pixels = kWidth * kHeight;
private:
VideoSinkInterface<VideoFrame>* sink_ = nullptr;
VideoAdapter adapter_{2};
};
struct Pipeline {
Source source;
std::unique_ptr<SendStatisticsProxy> stats;
std::unique_ptr<Sink> sink;
std::unique_ptr<VideoStreamEncoder> encoder;
TaskQueueBase* encode_queue = nullptr;
};
VideoEncoderConfig Config(const SdpVideoFormat& format) {
VideoEncoderConfig config;
config.codec_type = format.name == "H264" ? kVideoCodecH264 : kVideoCodecVP8;
config.video_format = format;
config.content_type = VideoEncoderConfig::ContentType::kRealtimeVideo;
config.number_of_streams = 1;
config.max_bitrate_bps = 2'000'000;
config.frame_drop_enabled = true;
config.is_quality_scaling_allowed = true;
config.simulcast_layers.resize(1);
config.simulcast_layers[0].active = true;
config.simulcast_layers[0].max_framerate = kFps;
config.simulcast_layers[0].max_bitrate_bps = 2'000'000;
config.simulcast_layers[0].num_temporal_layers = 1;
if (config.codec_type == kVideoCodecVP8) {
auto vp8 = VideoEncoder::GetDefaultVp8Settings();
vp8.automaticResizeOn = true;
config.encoder_specific_settings =
make_ref_counted<VideoEncoderConfig::Vp8EncoderSpecificSettings>(vp8);
}
return config;
}
void Start(Pipeline& pipeline, const Environment& env, PoolFactory& factory,
VideoBitrateAllocatorFactory* allocator, int consumer, SharedProof& proof) {
VideoSendStream::Config send_config(nullptr);
const auto format = factory.GetSupportedFormats().front();
send_config.rtp.ssrcs = {static_cast<uint32_t>(100 + consumer)};
send_config.rtp.payload_name = format.name;
send_config.rtp.payload_type = 96;
pipeline.stats = std::make_unique<SendStatisticsProxy>(
&env.clock(), send_config, VideoEncoderConfig::ContentType::kRealtimeVideo,
env.field_trials());
pipeline.sink = std::make_unique<Sink>(env, consumer, proof, format.name == "H264");
VideoStreamEncoderSettings settings(VideoEncoder::Capabilities(false));
settings.encoder_factory = &factory;
settings.bitrate_allocator_factory = allocator;
auto queue = env.task_queue_factory().CreateTaskQueue(
"PoolProbeEncoder", TaskQueueFactory::Priority::kNormal);
pipeline.encode_queue = queue.get();
auto cadence = FrameCadenceAdapterInterface::Create(
&env.clock(), queue.get(), nullptr, TaskQueueBase::Current(), env.field_trials());
pipeline.encoder = std::make_unique<VideoStreamEncoder>(
env, 1, pipeline.stats.get(), settings,
std::make_unique<OveruseFrameDetector>(env, pipeline.stats.get()),
std::move(cadence), std::move(queue),
VideoStreamEncoder::BitrateAllocationCallbackType::kVideoBitrateAllocation);
pipeline.encoder->SetSink(pipeline.sink.get(), false);
pipeline.encoder->SetSource(&pipeline.source, DegradationPreference::MAINTAIN_FRAMERATE);
pipeline.encoder->SetStartBitrate(2'000'000);
pipeline.encoder->ConfigureEncoder(Config(format), 1200);
pipeline.encoder->OnBitrateUpdated(DataRate::BitsPerSec(2'000'000),
DataRate::BitsPerSec(2'000'000), 0, 20, 0);
}
VideoFrame MakeFrame(const Environment& env, int index) {
auto pixels = I420Buffer::Create(kWidth, kHeight);
for (int y = 0; y < kHeight; ++y) {
for (int x = 0; x < kWidth; ++x) {
const int shifted = x + index * 5;
pixels->MutableDataY()[y * pixels->StrideY() + x] =
static_cast<uint8_t>(32 + ((shifted * 3 + y * 2 +
(((shifted / 23) ^ (y / 17)) & 7) * 19) % 192));
}
}
for (int y = 0; y < kHeight / 2; ++y) {
for (int x = 0; x < kWidth / 2; ++x) {
pixels->MutableDataU()[y * pixels->StrideU() + x] =
static_cast<uint8_t>(80 + (x + index * 2) % 96);
pixels->MutableDataV()[y * pixels->StrideV() + x] =
static_cast<uint8_t>(80 + (y + index) % 96);
}
}
return VideoFrame::Builder().set_video_frame_buffer(pixels)
.set_timestamp_us(env.clock().TimeInMicroseconds())
.set_ntp_time_ms(env.clock().CurrentNtpInMilliseconds())
.set_id(static_cast<uint16_t>(index)).build();
}
void PrintCounts(const Counts& before, const Counts& after, double seconds) {
std::cout << "{\"encoded\":" << after.encoded - before.encoded
<< ",\"decoded\":" << after.decoded - before.decoded
<< ",\"fps\":" << (after.decoded - before.decoded) / seconds
<< ",\"kbps\":" << (after.bytes - before.bytes) * 0.008 / seconds
<< ",\"errors\":" << after.errors - before.errors
<< ",\"keyframes\":" << after.keyframes - before.keyframes
<< ",\"width\":" << after.width << ",\"height\":" << after.height
<< ",\"minWidth\":" << after.min_width
<< ",\"minHeight\":" << after.min_height
<< ",\"firstFullMs\":" << after.first_full_ms
<< ",\"firstReducedMs\":" << after.first_reduced_ms << '}';
}
} // namespace
int main(int argc, char** argv) {
try {
const std::vector<std::string> args(argv + 1, argv + argc);
const auto has = [&](const char* flag) { return std::find(args.begin(), args.end(), flag) != args.end(); };
const bool pooled = has("--pooled");
const bool group_adjuster = has("--group-adjuster");
const bool no_adjuster = has("--no-adjuster") || group_adjuster;
const bool healthy_only = has("--healthy-only");
const bool realtime = has("--realtime");
const bool latency_probe = has("--latency-probe");
if (latency_probe && !realtime)
throw std::runtime_error("--latency-probe requires --realtime");
const bool shared_pipeline = has("--shared-pipeline");
const bool h264 = has("--h264");
if (h264 && (!shared_pipeline || !healthy_only || !has("--export")))
throw std::runtime_error("--h264 requires --shared-pipeline --healthy-only --export PATH");
if (shared_pipeline && (pooled || group_adjuster || no_adjuster || has("--unsafe-skip") ||
(!healthy_only && !latency_probe)))
throw std::runtime_error("shared pipeline control requires a healthy/latency trace with stock VSE settings");
const auto env = CreateEnvironment(std::make_unique<Trials>(no_adjuster));
const bool unsafe_skip = has("--unsafe-skip");
PoolFactory factory(pooled, group_adjuster, unsafe_skip,
h264 ? screener::probe::CreateHardwareEncoderFactory() : nullptr);
auto allocator = CreateBuiltinVideoBitrateAllocatorFactory();
auto worker = env.task_queue_factory().CreateTaskQueue(
"PoolProbeWorker", TaskQueueFactory::Priority::kNormal);
SharedProof proof;
std::array<Pipeline, 2> pipelines;
Sync(worker.get(), [&] {
Start(pipelines[0], env, factory, allocator.get(), 0, proof);
if (shared_pipeline) {
pipelines[1].sink = std::make_unique<Sink>(env, 1, proof, h264);
pipelines[0].sink->SetDuplicate(pipelines[1].sink.get());
} else Start(pipelines[1], env, factory, allocator.get(), 1, proof);
});
const auto export_arg = std::find(args.begin(), args.end(), "--export");
if (export_arg != args.end()) {
if (export_arg + 1 == args.end()) throw std::runtime_error("--export needs a path");
pipelines[0].sink->Export(*(export_arg + 1));
}
int frame_index = 0;
auto run = [&](const char* name, int seconds, int weak_bitrate, bool skip, bool retire) {
Sync(worker.get(), [&] {
if (retire) {
if (shared_pipeline) pipelines[0].sink->SetDuplicate(nullptr);
else {
pipelines[1].encoder->Stop();
pipelines[1].encoder.reset();
}
}
});
const auto a = pipelines[0].sink->Snapshot();
const auto b = pipelines[1].sink->Snapshot();
pipelines[0].sink->ResetMinimum();
pipelines[1].sink->ResetMinimum();
const int encodes = factory.encodes.load(), hits = factory.hits.load();
const auto started = std::chrono::steady_clock::now();
for (int frame = 0; frame < seconds * kFps; ++frame) {
// Equal feedback cadence for both controllers; only B's budget differs.
if (frame % 6 == 0) Sync(worker.get(), [&] {
for (int i = 0; i < 2; ++i) {
if (!pipelines[i].encoder) continue;
const auto rate = DataRate::BitsPerSec(i == 0 ? 2'000'000 : weak_bitrate);
pipelines[i].encoder->OnBitrateUpdated(rate, rate, 0, 20, 0);
}
});
auto original = MakeFrame(env, ++frame_index);
for (int consumer = 0; consumer < 2; ++consumer) {
if (!pipelines[consumer].encoder) continue;
if (consumer == 1 && (retire || (skip && frame == kFps))) continue;
Sync(worker.get(), [&, consumer] { pipelines[consumer].source.Push(original); });
if (!realtime) Sync(pipelines[consumer].encode_queue, [] {});
}
std::this_thread::sleep_until(started +
std::chrono::microseconds((frame + 1) * 1'000'000LL / kFps));
}
for (auto& pipeline : pipelines) {
if (pipeline.encoder) Sync(pipeline.encode_queue, [] {});
}
Sync(worker.get(), [] {});
const double elapsed = std::chrono::duration<double>(
std::chrono::steady_clock::now() - started).count();
std::cout << "{\"phase\":\"" << name << "\",\"seconds\":" << elapsed
<< ",\"encodes\":" << factory.encodes.load() - encodes
<< ",\"hits\":" << factory.hits.load() - hits << ",\"a\":";
PrintCounts(a, pipelines[0].sink->Snapshot(), elapsed);
std::cout << ",\"b\":";
PrintCounts(b, pipelines[1].sink->Snapshot(), elapsed);
const auto stats_a = pipelines[0].stats->GetStats();
const auto stats_b = shared_pipeline ? stats_a : pipelines[1].stats->GetStats();
std::cout << ",\"aEncodeMs\":" << stats_a.avg_encode_time_ms
<< ",\"bEncodeMs\":" << stats_b.avg_encode_time_ms
<< ",\"aEncodeUsage\":" << stats_a.encode_usage_percent
<< ",\"bEncodeUsage\":" << stats_b.encode_usage_percent
<< ",\"aCpuLimitedResolution\":" << (stats_a.cpu_limited_resolution ? "true" : "false")
<< ",\"bCpuLimitedResolution\":" << (stats_b.cpu_limited_resolution ? "true" : "false")
<< ",\"aCpuLimitedFramerate\":" << (stats_a.cpu_limited_framerate ? "true" : "false")
<< ",\"bCpuLimitedFramerate\":" << (stats_b.cpu_limited_framerate ? "true" : "false")
<< "}\n" << std::flush;
};
run("healthy", 6, 2'000'000, false, false);
if (latency_probe) {
factory.simulatedLatencyMs = 45;
run("encoder-delayed", 30, 2'000'000, false, false);
factory.simulatedLatencyMs = 0;
run("encoder-released", 30, 2'000'000, false, false);
} else if (!healthy_only) {
run("weak", 15, 100'000, false, false);
run("released", 25, 2'000'000, false, false);
}
if (!shared_pipeline) run("skipped-input", 3, 2'000'000, true, false);
run("retired-b", 2, 2'000'000, false, true);
Sync(worker.get(), [&] {
pipelines[0].encoder->Stop();
pipelines[0].encoder.reset();
});
pipelines[0].sink->FinishExport();
const auto a = pipelines[0].sink->Snapshot(), b = pipelines[1].sink->Snapshot();
const bool integrity = (h264 ? a.encoded > 0 && b.encoded > 0 : a.decoded > 0 && b.decoded > 0) && factory.live.load() == 0 &&
a.errors + b.errors + proof.mismatches.load() == 0 &&
(!pooled || factory.hits.load() > 0 && proof.compared.load() > 0) &&
(!shared_pipeline || proof.compared.load() > 0);
std::cout << "{\"integrity\":\"" << (integrity ? "pass" : "fail")
<< "\",\"codec\":\"" << (h264 ? "h264" : "vp8")
<< "\",\"pooled\":" << (pooled ? "true" : "false")
<< ",\"adjuster\":" << (no_adjuster ? "false" : "true")
<< ",\"groupAdjuster\":" << (group_adjuster ? "true" : "false")
<< ",\"unsafeSkip\":" << (unsafe_skip ? "true" : "false")
<< ",\"realtime\":" << (realtime ? "true" : "false")
<< ",\"latencyProbe\":" << (latency_probe ? "true" : "false")
<< ",\"sharedPipeline\":" << (shared_pipeline ? "true" : "false")
<< ",\"encodes\":" << factory.encodes.load()
<< ",\"hits\":" << factory.hits.load()
<< ",\"splits\":" << factory.splits.load()
<< ",\"rateSplits\":" << factory.rateSplits.load()
<< ",\"dependencySplits\":" << factory.dependencySplits.load()
<< ",\"created\":" << factory.created.load()
<< ",\"joins\":" << factory.joins.load()
<< ",\"live\":" << factory.live.load()
<< ",\"maxLive\":" << factory.maxLive.load()
<< ",\"callbacks\":" << factory.callbacks.load()
<< ",\"samePayloadCompared\":" << proof.compared.load()
<< ",\"decodedHashComparisons\":" << proof.decodedCompared.load()
<< ",\"decodedHashMismatches\":" << proof.mismatches.load()
<< ",\"aWantsUpdates\":" << pipelines[0].source.wants_updates
<< ",\"bWantsUpdates\":" << pipelines[1].source.wants_updates
<< ",\"aMinPixels\":" << pipelines[0].source.min_pixels
<< ",\"bMinPixels\":" << pipelines[1].source.min_pixels << "}\n";
return integrity ? 0 : 1;
} catch (const std::exception& error) {
std::cerr << "pool harness failed: " << error.what() << '\n';
return 2;
}
}
native/probes/encoder-pool/pool.h
+316
View File
@@ -0,0 +1,316 @@
#pragma once
#include <algorithm>
#include <atomic>
#include <chrono>
#include <memory>
#include <iostream>
#include <mutex>
#include <optional>
#include <thread>
#include <vector>
#include "api/environment/environment.h"
#include "api/video_codecs/builtin_video_encoder_factory.h"
#include "api/video_codecs/video_encoder_factory.h"
#include "modules/video_coding/include/video_codec_interface.h"
#include "rtc_base/checks.h"
#include "video/encoder_bitrate_adjuster.h"
// A one-source, single-stream experiment, not a product encoder registry.
// H264 uses the shared-pipeline control, not per-sender cache adaptation.
// Matching requests share only one last completed access unit. No warm cache.
// ponytail: one mutex serializes groups; validate per-group workers before production.
class PoolFactory final : public webrtc::VideoEncoderFactory {
public:
explicit PoolFactory(bool pooled, bool group_adjuster = false, bool unsafe_skip = false,
std::unique_ptr<webrtc::VideoEncoderFactory> backend = nullptr)
: pooled_(pooled), group_adjuster_(group_adjuster), unsafe_skip_(unsafe_skip),
builtin_(backend ? std::move(backend) : webrtc::CreateBuiltinVideoEncoderFactory()),
format_(builtin_->GetSupportedFormats().front()) {}
std::atomic<int> encodes{0}, hits{0}, splits{0}, live{0}, maxLive{0}, callbacks{0};
std::atomic<int> dependencySplits{0}, rateSplits{0};
std::atomic<int> created{0}, joins{0};
std::atomic<int> simulatedLatencyMs{0};
std::vector<webrtc::SdpVideoFormat> GetSupportedFormats() const override {
return {format_};
}
private:
struct Group final : webrtc::EncodedImageCallback {
PoolFactory& owner;
webrtc::VideoCodec config;
webrtc::VideoEncoder::Settings settings;
std::unique_ptr<webrtc::VideoEncoder> codec;
std::optional<webrtc::VideoEncoder::RateControlParameters> rates;
std::optional<webrtc::VideoEncoder::RateControlParameters> applied_rates;
std::unique_ptr<webrtc::EncoderBitrateAdjuster> adjuster;
float loss = 0;
int64_t rtt = 0;
std::optional<uint32_t> timestamp, previous, attempted;
std::optional<webrtc::EncodedImage> image;
webrtc::CodecSpecificInfo info;
bool dropped = false;
bool pendingKey = false;
Group(PoolFactory& parent, const webrtc::Environment& env,
const webrtc::VideoCodec& configuration,
const webrtc::VideoEncoder::Settings& options)
: owner(parent), config(configuration), settings(options),
codec(parent.builtin_->Create(env, parent.format_)) {
RTC_CHECK(codec);
RTC_CHECK_EQ(codec->InitEncode(&config, settings), 0);
codec->RegisterEncodeCompleteCallback(this);
if (owner.group_adjuster_) {
adjuster = std::make_unique<webrtc::EncoderBitrateAdjuster>(config, env.field_trials(), env.clock());
adjuster->OnEncoderInfo(codec->GetEncoderInfo());
}
++owner.created;
const int count = ++owner.live;
owner.maxLive.store(std::max(owner.maxLive.load(), count));
}
~Group() override {
codec->RegisterEncodeCompleteCallback(nullptr);
codec->Release();
--owner.live;
}
Result OnEncodedImage(const webrtc::EncodedImage& output,
const webrtc::CodecSpecificInfo* details) override {
RTC_CHECK(!image.has_value());
image = output;
if (details) info = *details;
if (adjuster) adjuster->OnEncodedFrame(webrtc::DataSize::Bytes(output.size()), 0, 0);
return Result(Result::OK);
}
void OnFrameDropped(uint32_t, int, bool) override {
dropped = true;
}
void UpdateCodecRates() {
if (!rates) return;
auto adjusted = *rates;
if (adjuster) adjusted.bitrate = adjuster->AdjustRateAllocation(*rates);
if (applied_rates != adjusted) codec->SetRates(adjusted);
applied_rates = adjusted;
}
};
static bool SameConfig(const Group& group, const webrtc::VideoCodec& c,
const webrtc::VideoEncoder::Settings& s) {
const auto& a = group.config;
return a.width == c.width && a.height == c.height &&
a.startBitrate == c.startBitrate && a.maxBitrate == c.maxBitrate &&
a.minBitrate == c.minBitrate && a.maxFramerate == c.maxFramerate &&
a.qpMax == c.qpMax && a.mode == c.mode && a.active == c.active &&
a.GetFrameDropEnabled() == c.GetFrameDropEnabled() &&
a.GetVideoEncoderComplexity() == c.GetVideoEncoderComplexity() &&
a.GetScalabilityMode() == c.GetScalabilityMode() && a.codecType == c.codecType &&
(a.codecType == webrtc::kVideoCodecVP8 ? a.VP8() == c.VP8() : a.H264() == c.H264()) &&
group.settings.number_of_cores == s.number_of_cores &&
group.settings.max_payload_size == s.max_payload_size &&
group.settings.encoder_thread_limit == s.encoder_thread_limit;
}
class Encoder final : public webrtc::VideoEncoder {
public:
Encoder(PoolFactory& owner, const webrtc::Environment& env)
: owner_(owner), env_(env),
metadata_(owner.builtin_->Create(env, owner.format_)) {}
~Encoder() override { Release(); }
int InitEncode(const webrtc::VideoCodec* c, const Settings& s) override {
RTC_CHECK(c->codecType == webrtc::kVideoCodecVP8 || c->codecType == webrtc::kVideoCodecH264);
RTC_CHECK_LE(c->numberOfSimulcastStreams, 1);
RTC_CHECK_EQ(c->codecType == webrtc::kVideoCodecVP8 ? c->VP8().numberOfTemporalLayers : c->H264().numberOfTemporalLayers, 1);
RTC_CHECK_EQ(c->mode, webrtc::VideoCodecMode::kRealtimeVideo);
std::lock_guard lock(owner_.mutex_);
group_.reset();
last_.reset();
config_ = *c;
settings_ = s;
rates_.reset();
group_ = FindOrCreate(false, 0);
return 0;
}
int32_t RegisterEncodeCompleteCallback(webrtc::EncodedImageCallback* callback) override {
callback_ = callback;
return 0;
}
int32_t Release() override {
std::lock_guard lock(owner_.mutex_);
group_.reset();
last_.reset();
return 0;
}
void SetRates(const RateControlParameters& rates) override { rates_ = rates; }
void OnPacketLossRateUpdate(float loss) override { loss_ = loss; }
void OnRttUpdate(int64_t rtt) override { rtt_ = rtt; }
EncoderInfo GetEncoderInfo() const override {
std::lock_guard lock(owner_.mutex_);
return group_ ? group_->codec->GetEncoderInfo() : metadata_->GetEncoderInfo();
}
int32_t Encode(const webrtc::VideoFrame& frame,
const std::vector<webrtc::VideoFrameType>* requested) override {
std::optional<webrtc::EncodedImage> output;
webrtc::CodecSpecificInfo info;
bool dropped = false;
int result = 0;
{
std::lock_guard lock(owner_.mutex_);
const uint32_t ts = frame.rtp_timestamp();
bool key = requested && std::find(requested->begin(), requested->end(),
webrtc::VideoFrameType::kVideoFrameKey) != requested->end();
const auto before = group_;
if (!Compatible(*group_)) {
// Single-owner rate changes must retain the encoder's adaptation history.
if (group_.use_count() == 2) {
ApplyRates(*group_);
} else {
if (owner_.rateSplits.load() < 8 && group_->rates && rates_) {
std::cerr << "rate split " << group_->rates->bitrate.get_sum_bps()
<< '/' << rates_->bitrate.get_sum_bps() << " fps "
<< group_->rates->framerate_fps << '/' << rates_->framerate_fps
<< " target " << group_->rates->target_bitrate.get_sum_bps()
<< '/' << rates_->target_bitrate.get_sum_bps()
<< " allocation " << group_->rates->bandwidth_allocation.bps()
<< '/' << rates_->bandwidth_allocation.bps()
<< " rtt " << group_->rtt << '/' << rtt_
<< " loss " << group_->loss << '/' << loss_ << '\n';
}
group_.reset();
group_ = FindOrCreate(true, ts);
++owner_.rateSplits;
}
}
if (owner_.pooled_) {
for (auto& reference : owner_.groups_) {
const auto candidate = reference.lock();
if (candidate && candidate != group_ && Compatible(*candidate)) {
if (candidate->attempted == ts && candidate->image &&
candidate->image->_frameType == webrtc::VideoFrameType::kVideoFrameKey) {
group_ = candidate;
++owner_.joins;
break;
}
// Demand has reconverged; request normal keyframe recovery once.
candidate->pendingKey = true;
}
}
}
bool cached = group_->attempted == ts;
bool continuous = cached ? last_ == group_->previous : last_ == group_->timestamp;
if (!owner_.unsafe_skip_ && !continuous && !(cached && group_->image &&
group_->image->_frameType == webrtc::VideoFrameType::kVideoFrameKey)) {
group_.reset();
group_ = FindOrCreate(true, ts);
cached = group_->attempted == ts;
key = true;
++owner_.dependencySplits;
}
if (key && cached && group_->image &&
group_->image->_frameType != webrtc::VideoFrameType::kVideoFrameKey) {
group_.reset();
group_ = NewGroup();
cached = false;
}
if (before != group_) {
++owner_.splits;
last_.reset();
key = true;
}
if (cached) {
++owner_.hits;
} else {
key = key || group_->pendingKey;
group_->pendingKey = false;
group_->previous = group_->timestamp;
group_->attempted = ts;
group_->image.reset();
group_->dropped = false;
const std::vector<webrtc::VideoFrameType> types{
key ? webrtc::VideoFrameType::kVideoFrameKey : webrtc::VideoFrameType::kVideoFrameDelta};
++owner_.encodes;
group_->UpdateCodecRates();
const int64_t latency_us = static_cast<int64_t>(owner_.simulatedLatencyMs.load()) *
1000 * group_->config.width * group_->config.height / (640 * 360);
if (latency_us > 0) std::this_thread::sleep_for(std::chrono::microseconds(latency_us));
result = group_->codec->Encode(frame, &types);
if (group_->image) group_->timestamp = ts;
}
output = group_->image;
info = group_->info;
dropped = group_->dropped;
if (output) last_ = ts;
}
if (output) {
output->SetRtpTimestamp(frame.rtp_timestamp());
++owner_.callbacks;
callback_->OnEncodedImage(*output, &info);
} else if (dropped) {
callback_->OnFrameDropped(frame.rtp_timestamp(), 0, true);
}
return result;
}
private:
bool Compatible(const Group& group) const {
// Upstream DefaultTemporalLayers ignores RTT/loss for this L1T1 codec.
return SameConfig(group, *config_, *settings_) && group.rates == rates_;
}
void ApplyRates(Group& group) {
group.rates = rates_;
group.UpdateCodecRates();
group.codec->OnPacketLossRateUpdate(loss_);
group.codec->OnRttUpdate(rtt_);
group.loss = loss_;
group.rtt = rtt_;
}
std::shared_ptr<Group> NewGroup() {
std::erase_if(owner_.groups_, [](const auto& group) { return group.expired(); });
auto group = std::make_shared<Group>(owner_, env_, *config_, *settings_);
ApplyRates(*group);
owner_.groups_.push_back(group);
return group;
}
std::shared_ptr<Group> FindOrCreate(bool encoding, uint32_t ts) {
if (owner_.pooled_) {
for (auto it = owner_.groups_.begin(); it != owner_.groups_.end();) {
auto group = it->lock();
if (!group) { it = owner_.groups_.erase(it); continue; }
++it;
if (!Compatible(*group)) continue;
if (!encoding && !group->timestamp) return group;
if (encoding && group->attempted == ts && group->image &&
group->image->_frameType == webrtc::VideoFrameType::kVideoFrameKey) return group;
if (encoding && !group->timestamp) return group;
}
}
return NewGroup();
}
PoolFactory& owner_;
webrtc::Environment env_;
std::unique_ptr<webrtc::VideoEncoder> metadata_;
std::optional<webrtc::VideoCodec> config_;
std::optional<Settings> settings_;
std::optional<RateControlParameters> rates_;
float loss_ = 0;
int64_t rtt_ = 0;
std::shared_ptr<Group> group_;
std::optional<uint32_t> last_;
webrtc::EncodedImageCallback* callback_ = nullptr;
};
public:
std::unique_ptr<webrtc::VideoEncoder> Create(
const webrtc::Environment& env, const webrtc::SdpVideoFormat&) override {
return std::make_unique<Encoder>(*this, env);
}
private:
bool pooled_;
bool group_adjuster_;
bool unsafe_skip_;
std::unique_ptr<webrtc::VideoEncoderFactory> builtin_;
webrtc::SdpVideoFormat format_;
std::mutex mutex_;
std::vector<std::weak_ptr<Group>> groups_;
};