mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-13 13:50:40 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Create and initialize encoders only for active streams

Browse source 
Bug: webrtc:12407
Change-Id: Id30fcb84dcbfffa30c7a34b15564ab5049cec96c
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/204066
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#33141}
This commit is contained in:
Sergey Silkin 2021-02-02 18:23:40 +01:00 committed by Commit Bot
parent 312ea0e144
commit 0e3cb9fb20
6 changed files with 491 additions and 337 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
media/BUILD.gn
View file

@ -576,6 +576,7 @@ if (rtc_include_tests) {
"../api/units:time_delta", "../api/units:time_delta",
"../api/video:builtin_video_bitrate_allocator_factory", "../api/video:builtin_video_bitrate_allocator_factory",
"../api/video:video_bitrate_allocation", "../api/video:video_bitrate_allocation",
"../api/video:video_codec_constants",
"../api/video:video_frame", "../api/video:video_frame",
"../api/video:video_rtp_headers", "../api/video:video_rtp_headers",
"../api/video_codecs:builtin_video_decoder_factory", "../api/video_codecs:builtin_video_decoder_factory",

9
media/engine/encoder_simulcast_proxy_unittest.cc
View file

@ -49,7 +49,8 @@ TEST(EncoderSimulcastProxy, ChoosesCorrectImplementation) {
2000, 2000,
1000, 1000,
1000, 1000,
56}; 56,
true};
codec_settings.simulcastStream[1] = {test::kTestWidth, codec_settings.simulcastStream[1] = {test::kTestWidth,
test::kTestHeight, test::kTestHeight,
test::kTestFrameRate, test::kTestFrameRate,
@ -57,7 +58,8 @@ TEST(EncoderSimulcastProxy, ChoosesCorrectImplementation) {
3000, 3000,
1000, 1000,
1000, 1000,
56}; 56,
true};
codec_settings.simulcastStream[2] = {test::kTestWidth, codec_settings.simulcastStream[2] = {test::kTestWidth,
test::kTestHeight, test::kTestHeight,
test::kTestFrameRate, test::kTestFrameRate,
@ -65,7 +67,8 @@ TEST(EncoderSimulcastProxy, ChoosesCorrectImplementation) {
5000, 5000,
1000, 1000,
1000, 1000,
56}; 56,
true};
codec_settings.numberOfSimulcastStreams = 3; codec_settings.numberOfSimulcastStreams = 3;
auto mock_encoder = std::make_unique<NiceMock<MockVideoEncoder>>(); auto mock_encoder = std::make_unique<NiceMock<MockVideoEncoder>>();

624
media/engine/simulcast_encoder_adapter.cc
View file

@ -62,32 +62,29 @@ uint32_t SumStreamMaxBitrate(int streams, const webrtc::VideoCodec& codec) {
return bitrate_sum; return bitrate_sum;
} }
int NumberOfStreams(const webrtc::VideoCodec& codec) { int CountAllStreams(const webrtc::VideoCodec& codec) {
int streams = int total_streams_count =
codec.numberOfSimulcastStreams < 1 ? 1 : codec.numberOfSimulcastStreams; codec.numberOfSimulcastStreams < 1 ? 1 : codec.numberOfSimulcastStreams;
uint32_t simulcast_max_bitrate = SumStreamMaxBitrate(streams, codec); uint32_t simulcast_max_bitrate =
SumStreamMaxBitrate(total_streams_count, codec);
if (simulcast_max_bitrate == 0) { if (simulcast_max_bitrate == 0) {
streams = 1; total_streams_count = 1;
} }
return streams; return total_streams_count;
} }
struct StreamDimensions { int CountActiveStreams(const webrtc::VideoCodec& codec) {
size_t num_active_streams; if (codec.numberOfSimulcastStreams < 1) {
size_t first_active_stream_idx; return 1;
}; }
StreamDimensions ActiveStreams(const webrtc::VideoCodec& codec) { int total_streams_count = CountAllStreams(codec);
size_t num_configured_streams = NumberOfStreams(codec); int active_streams_count = 0;
StreamDimensions dimensions{0, 0}; for (int i = 0; i < total_streams_count; ++i) {
for (size_t i = 0; i < num_configured_streams; ++i) {
if (codec.simulcastStream[i].active) { if (codec.simulcastStream[i].active) {
++dimensions.num_active_streams; ++active_streams_count;
if (dimensions.num_active_streams == 1) {
dimensions.first_active_stream_idx = i;
}
} }
} }
return dimensions; return active_streams_count;
} }
int VerifyCodec(const webrtc::VideoCodec* inst) { int VerifyCodec(const webrtc::VideoCodec* inst) {
@ -105,80 +102,119 @@ int VerifyCodec(const webrtc::VideoCodec* inst) {
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
} }
if (inst->codecType == webrtc::kVideoCodecVP8 && if (inst->codecType == webrtc::kVideoCodecVP8 &&
inst->VP8().automaticResizeOn && inst->VP8().automaticResizeOn && CountActiveStreams(*inst) > 1) {
ActiveStreams(*inst).num_active_streams > 1) {
return WEBRTC_VIDEO_CODEC_ERR_PARAMETER; return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
} }
return WEBRTC_VIDEO_CODEC_OK; return WEBRTC_VIDEO_CODEC_OK;
} }
bool StreamResolutionCompare(const webrtc::SpatialLayer& a, bool StreamQualityCompare(const webrtc::SpatialLayer& a,
const webrtc::SpatialLayer& b) { const webrtc::SpatialLayer& b) {
return std::tie(a.height, a.width, a.maxBitrate, a.maxFramerate) < return std::tie(a.height, a.width, a.maxBitrate, a.maxFramerate) <
std::tie(b.height, b.width, b.maxBitrate, b.maxFramerate); std::tie(b.height, b.width, b.maxBitrate, b.maxFramerate);
} }
void GetLowestAndHighestQualityStreamIndixes(
rtc::ArrayView<webrtc::SpatialLayer> streams,
int* lowest_quality_stream_idx,
int* highest_quality_stream_idx) {
const auto lowest_highest_quality_streams =
absl::c_minmax_element(streams, StreamQualityCompare);
*lowest_quality_stream_idx =
std::distance(streams.begin(), lowest_highest_quality_streams.first);
*highest_quality_stream_idx =
std::distance(streams.begin(), lowest_highest_quality_streams.second);
}
std::vector<uint32_t> GetStreamStartBitratesKbps(
const webrtc::VideoCodec& codec) {
std::vector<uint32_t> start_bitrates;
std::unique_ptr<webrtc::VideoBitrateAllocator> rate_allocator =
std::make_unique<webrtc::SimulcastRateAllocator>(codec);
webrtc::VideoBitrateAllocation allocation =
rate_allocator->Allocate(webrtc::VideoBitrateAllocationParameters(
codec.startBitrate * 1000, codec.maxFramerate));
int total_streams_count = CountAllStreams(codec);
for (int i = 0; i < total_streams_count; ++i) {
uint32_t stream_bitrate = allocation.GetSpatialLayerSum(i) / 1000;
start_bitrates.push_back(stream_bitrate);
}
return start_bitrates;
}
} // namespace } // namespace
namespace webrtc { namespace webrtc {
SimulcastEncoderAdapter::EncoderContext::EncoderContext( SimulcastEncoderAdapter::EncoderContext::EncoderContext(
SimulcastEncoderAdapter* parent,
std::unique_ptr<VideoEncoder> encoder, std::unique_ptr<VideoEncoder> encoder,
std::unique_ptr<FramerateController> framerate_controller, bool prefer_temporal_support)
int stream_idx, : encoder_(std::move(encoder)),
uint16_t width, prefer_temporal_support_(prefer_temporal_support) {}
uint16_t height,
bool send_stream)
: parent_(parent),
encoder_(std::move(encoder)),
framerate_controller_(std::move(framerate_controller)),
stream_idx_(stream_idx),
width_(width),
height_(height),
needs_keyframe_(false),
send_stream_(send_stream) {
if (parent) {
encoder_->RegisterEncodeCompleteCallback(this);
}
}
SimulcastEncoderAdapter::EncoderContext::EncoderContext(EncoderContext&& rhs) void SimulcastEncoderAdapter::EncoderContext::Release() {
: parent_(rhs.parent_),
encoder_(std::move(rhs.encoder_)),
framerate_controller_(std::move(rhs.framerate_controller_)),
stream_idx_(rhs.stream_idx_),
width_(rhs.width_),
height_(rhs.height_),
needs_keyframe_(rhs.needs_keyframe_),
send_stream_(rhs.send_stream_) {
if (parent_) {
encoder_->RegisterEncodeCompleteCallback(this);
}
}
SimulcastEncoderAdapter::EncoderContext::~EncoderContext() {
if (encoder_) { if (encoder_) {
encoder_->RegisterEncodeCompleteCallback(nullptr); encoder_->RegisterEncodeCompleteCallback(nullptr);
encoder_->Release(); encoder_->Release();
} }
} }
std::unique_ptr<VideoEncoder> SimulcastEncoderAdapter::StreamContext::StreamContext(
SimulcastEncoderAdapter::EncoderContext::Release() && { SimulcastEncoderAdapter* parent,
encoder_->RegisterEncodeCompleteCallback(nullptr); std::unique_ptr<EncoderContext> encoder_context,
encoder_->Release(); std::unique_ptr<FramerateController> framerate_controller,
return std::move(encoder_); int stream_idx,
uint16_t width,
uint16_t height,
bool is_paused)
: parent_(parent),
encoder_context_(std::move(encoder_context)),
framerate_controller_(std::move(framerate_controller)),
stream_idx_(stream_idx),
width_(width),
height_(height),
is_keyframe_needed_(false),
is_paused_(is_paused) {
if (parent_) {
encoder_context_->encoder().RegisterEncodeCompleteCallback(this);
}
} }
void SimulcastEncoderAdapter::EncoderContext::OnKeyframe(Timestamp timestamp) { SimulcastEncoderAdapter::StreamContext::StreamContext(StreamContext&& rhs)
needs_keyframe_ = false; : parent_(rhs.parent_),
encoder_context_(std::move(rhs.encoder_context_)),
framerate_controller_(std::move(rhs.framerate_controller_)),
stream_idx_(rhs.stream_idx_),
width_(rhs.width_),
height_(rhs.height_),
is_keyframe_needed_(rhs.is_keyframe_needed_),
is_paused_(rhs.is_paused_) {
if (parent_) {
encoder_context_->encoder().RegisterEncodeCompleteCallback(this);
}
}
SimulcastEncoderAdapter::StreamContext::~StreamContext() {
if (encoder_context_) {
encoder_context_->Release();
}
}
std::unique_ptr<SimulcastEncoderAdapter::EncoderContext>
SimulcastEncoderAdapter::StreamContext::ReleaseEncoderContext() && {
encoder_context_->Release();
return std::move(encoder_context_);
}
void SimulcastEncoderAdapter::StreamContext::OnKeyframe(Timestamp timestamp) {
is_keyframe_needed_ = false;
if (framerate_controller_) { if (framerate_controller_) {
framerate_controller_->AddFrame(timestamp.ms()); framerate_controller_->AddFrame(timestamp.ms());
} }
} }
bool SimulcastEncoderAdapter::EncoderContext::ShouldDropFrame( bool SimulcastEncoderAdapter::StreamContext::ShouldDropFrame(
Timestamp timestamp) { Timestamp timestamp) {
if (!framerate_controller_) { if (!framerate_controller_) {
return false; return false;
@ -192,7 +228,7 @@ bool SimulcastEncoderAdapter::EncoderContext::ShouldDropFrame(
} }
EncodedImageCallback::Result EncodedImageCallback::Result
SimulcastEncoderAdapter::EncoderContext::OnEncodedImage( SimulcastEncoderAdapter::StreamContext::OnEncodedImage(
const EncodedImage& encoded_image, const EncodedImage& encoded_image,
const CodecSpecificInfo* codec_specific_info) { const CodecSpecificInfo* codec_specific_info) {
RTC_CHECK(parent_); // If null, this method should never be called. RTC_CHECK(parent_); // If null, this method should never be called.
@ -200,7 +236,7 @@ SimulcastEncoderAdapter::EncoderContext::OnEncodedImage(
codec_specific_info); codec_specific_info);
} }
void SimulcastEncoderAdapter::EncoderContext::OnDroppedFrame( void SimulcastEncoderAdapter::StreamContext::OnDroppedFrame(
DropReason /*reason*/) { DropReason /*reason*/) {
RTC_CHECK(parent_); // If null, this method should never be called. RTC_CHECK(parent_); // If null, this method should never be called.
parent_->OnDroppedFrame(stream_idx_); parent_->OnDroppedFrame(stream_idx_);
@ -218,9 +254,9 @@ SimulcastEncoderAdapter::SimulcastEncoderAdapter(
primary_encoder_factory_(primary_factory), primary_encoder_factory_(primary_factory),
fallback_encoder_factory_(fallback_factory), fallback_encoder_factory_(fallback_factory),
video_format_(format), video_format_(format),
total_streams_count_(0),
bypass_mode_(false),
encoded_complete_callback_(nullptr), encoded_complete_callback_(nullptr),
first_active_stream_idx_(0),
num_active_streams_(0),
experimental_boosted_screenshare_qp_(GetScreenshareBoostedQpValue()), experimental_boosted_screenshare_qp_(GetScreenshareBoostedQpValue()),
boost_base_layer_quality_(RateControlSettings::ParseFromFieldTrials() boost_base_layer_quality_(RateControlSettings::ParseFromFieldTrials()
.Vp8BoostBaseLayerQuality()), .Vp8BoostBaseLayerQuality()),
@ -246,15 +282,15 @@ void SimulcastEncoderAdapter::SetFecControllerOverride(
int SimulcastEncoderAdapter::Release() { int SimulcastEncoderAdapter::Release() {
RTC_DCHECK_RUN_ON(&encoder_queue_); RTC_DCHECK_RUN_ON(&encoder_queue_);
while (!encoder_contexts_.empty()) { while (!stream_contexts_.empty()) {
// Move the encoder instances and put it on the |stored_encoders_| where it // Move the encoder instances and put it on the |cached_encoder_contexts_|
// it may possibly be reused from (ordering does not matter). // where it may possibly be reused from (ordering does not matter).
stored_encoders_.push(std::move(encoder_contexts_.back()).Release()); cached_encoder_contexts_.push_front(
encoder_contexts_.pop_back(); std::move(stream_contexts_.back()).ReleaseEncoderContext());
stream_contexts_.pop_back();
} }
num_active_streams_ = 0; bypass_mode_ = false;
first_active_stream_idx_ = 0;
// It's legal to move the encoder to another queue now. // It's legal to move the encoder to another queue now.
encoder_queue_.Detach(); encoder_queue_.Detach();
@ -264,7 +300,6 @@ int SimulcastEncoderAdapter::Release() {
return WEBRTC_VIDEO_CODEC_OK; return WEBRTC_VIDEO_CODEC_OK;
} }
// TODO(eladalon): s/inst/codec_settings/g.
int SimulcastEncoderAdapter::InitEncode( int SimulcastEncoderAdapter::InitEncode(
const VideoCodec* inst, const VideoCodec* inst,
const VideoEncoder::Settings& settings) { const VideoEncoder::Settings& settings) {
@ -279,137 +314,114 @@ int SimulcastEncoderAdapter::InitEncode(
return ret; return ret;
} }
ret = Release(); Release();
if (ret < 0) {
return ret;
}
int number_of_streams = NumberOfStreams(*inst);
RTC_DCHECK_LE(number_of_streams, kMaxSimulcastStreams);
bool doing_simulcast_using_adapter = (number_of_streams > 1);
auto active_streams = ActiveStreams(*inst);
num_active_streams_ = active_streams.num_active_streams;
first_active_stream_idx_ = active_streams.first_active_stream_idx;
codec_ = *inst; codec_ = *inst;
std::unique_ptr<VideoBitrateAllocator> rate_allocator = total_streams_count_ = CountAllStreams(*inst);
std::make_unique<SimulcastRateAllocator>(codec_);
VideoBitrateAllocation allocation = // TODO(ronghuawu): Remove once this is handled in LibvpxVp8Encoder.
rate_allocator->Allocate(VideoBitrateAllocationParameters( if (codec_.qpMax < kDefaultMinQp) {
codec_.startBitrate * 1000, codec_.maxFramerate)); codec_.qpMax = kDefaultMaxQp;
std::vector<uint32_t> start_bitrates;
for (int i = 0; i < kMaxSimulcastStreams; ++i) {
uint32_t stream_bitrate = allocation.GetSpatialLayerSum(i) / 1000;
start_bitrates.push_back(stream_bitrate);
} }
// Create |number_of_streams| of encoder instances and init them. bool is_legacy_singlecast = codec_.numberOfSimulcastStreams == 0;
auto spatial_layers = int lowest_quality_stream_idx = 0;
rtc::ArrayView<SpatialLayer>(codec_.simulcastStream, number_of_streams); int highest_quality_stream_idx = 0;
const auto minmax = if (!is_legacy_singlecast) {
absl::c_minmax_element(spatial_layers, StreamResolutionCompare); GetLowestAndHighestQualityStreamIndixes(
const auto lowest_resolution_stream_index = rtc::ArrayView<SpatialLayer>(codec_.simulcastStream,
minmax.first - spatial_layers.begin(); total_streams_count_),
const auto highest_resolution_stream_index = &lowest_quality_stream_idx, &highest_quality_stream_idx);
minmax.second - spatial_layers.begin(); }
RTC_DCHECK_LT(lowest_resolution_stream_index, number_of_streams); std::unique_ptr<EncoderContext> encoder_context = FetchOrCreateEncoderContext(
RTC_DCHECK_LT(highest_resolution_stream_index, number_of_streams); /*is_lowest_quality_stream=*/(
is_legacy_singlecast ||
codec_.simulcastStream[lowest_quality_stream_idx].active));
if (encoder_context == nullptr) {
return WEBRTC_VIDEO_CODEC_MEMORY;
}
for (int i = 0; i < number_of_streams; ++i) { // Two distinct scenarios:
// If an existing encoder instance exists, reuse it. // * Singlecast (total_streams_count == 1) or simulcast with simulcast-capable
// TODO(brandtr): Set initial RTP state (e.g., picture_id/tl0_pic_idx) here, // underlaying encoder implementation. SEA operates in bypass mode: original
// when we start storing that state outside the encoder wrappers. // settings are passed to the underlaying encoder, frame encode complete
std::unique_ptr<VideoEncoder> encoder; // callback is not intercepted.
if (!stored_encoders_.empty()) { // * Multi-encoder simulcast or singlecast if layers are deactivated
encoder = std::move(stored_encoders_.top()); // (total_streams_count > 1 and active_streams_count >= 1). SEA creates
stored_encoders_.pop(); // N=active_streams_count encoders and configures each to produce a single
} else { // stream.
encoder = primary_encoder_factory_->CreateVideoEncoder(video_format_);
if (fallback_encoder_factory_ != nullptr) { // Singlecast or simulcast with simulcast-capable underlaying encoder.
encoder = CreateVideoEncoderSoftwareFallbackWrapper( if (total_streams_count_ == 1 ||
fallback_encoder_factory_->CreateVideoEncoder(video_format_), encoder_context->encoder().GetEncoderInfo().supports_simulcast) {
std::move(encoder), int ret = encoder_context->encoder().InitEncode(&codec_, settings);
i == lowest_resolution_stream_index && if (ret >= 0) {
prefer_temporal_support_on_base_layer_); int active_streams_count = CountActiveStreams(*inst);
} stream_contexts_.emplace_back(
/*parent=*/nullptr, std::move(encoder_context),
/*framerate_controller=*/nullptr, /*stream_idx=*/0, codec_.width,
codec_.height, /*is_paused=*/active_streams_count == 0);
bypass_mode_ = true;
DestroyStoredEncoders();
rtc::AtomicOps::ReleaseStore(&inited_, 1);
return WEBRTC_VIDEO_CODEC_OK;
} }
bool encoder_initialized = false; encoder_context->Release();
if (doing_simulcast_using_adapter && i == 0 && if (total_streams_count_ == 1) {
encoder->GetEncoderInfo().supports_simulcast) { // Failed to initialize singlecast encoder.
ret = encoder->InitEncode(&codec_, settings); return ret;
if (ret < 0) { }
encoder->Release(); }
} else {
doing_simulcast_using_adapter = false; // Multi-encoder simulcast or singlecast (deactivated layers).
number_of_streams = 1; std::vector<uint32_t> stream_start_bitrate_kbps =
encoder_initialized = true; GetStreamStartBitratesKbps(codec_);
}
for (int stream_idx = 0; stream_idx < total_streams_count_; ++stream_idx) {
if (!is_legacy_singlecast && !codec_.simulcastStream[stream_idx].active) {
continue;
} }
VideoCodec stream_codec; if (encoder_context == nullptr) {
uint32_t start_bitrate_kbps = start_bitrates[i]; encoder_context = FetchOrCreateEncoderContext(
const bool send_stream = doing_simulcast_using_adapter /*is_lowest_quality_stream=*/stream_idx == lowest_quality_stream_idx);
? start_bitrate_kbps > 0 }
: num_active_streams_ > 0; if (encoder_context == nullptr) {
if (!doing_simulcast_using_adapter) { Release();
stream_codec = codec_; return WEBRTC_VIDEO_CODEC_MEMORY;
stream_codec.numberOfSimulcastStreams =
std::max<uint8_t>(1, stream_codec.numberOfSimulcastStreams);
} else {
// Cap start bitrate to the min bitrate in order to avoid strange codec
// behavior. Since sending will be false, this should not matter.
StreamResolution stream_resolution =
i == highest_resolution_stream_index
? StreamResolution::HIGHEST
: i == lowest_resolution_stream_index ? StreamResolution::LOWEST
: StreamResolution::OTHER;
start_bitrate_kbps =
std::max(spatial_layers[i].minBitrate, start_bitrate_kbps);
PopulateStreamCodec(codec_, i, start_bitrate_kbps, stream_resolution,
&stream_codec);
} }
// TODO(ronghuawu): Remove once this is handled in LibvpxVp8Encoder. VideoCodec stream_codec = MakeStreamCodec(
if (stream_codec.qpMax < kDefaultMinQp) { codec_, stream_idx, stream_start_bitrate_kbps[stream_idx],
stream_codec.qpMax = kDefaultMaxQp; /*is_lowest_quality_stream=*/stream_idx == lowest_quality_stream_idx,
/*is_highest_quality_stream=*/stream_idx == highest_quality_stream_idx);
int ret = encoder_context->encoder().InitEncode(&stream_codec, settings);
if (ret < 0) {
encoder_context.reset();
Release();
return ret;
} }
if (!encoder_initialized) { // Intercept frame encode complete callback only for upper streams, where
ret = encoder->InitEncode(&stream_codec, settings); // we need to set a correct stream index. Set |parent| to nullptr for the
if (ret < 0) { // lowest stream to bypass the callback.
// Explicitly destroy the current encoder; because we haven't registered SimulcastEncoderAdapter* parent = stream_idx > 0 ? this : nullptr;
// a StreamInfo for it yet, Release won't do anything about it.
encoder.reset();
Release();
return ret;
}
}
if (!doing_simulcast_using_adapter) { bool is_paused = stream_start_bitrate_kbps[stream_idx] == 0;
// Without simulcast, let the encoder call callbacks and do frame stream_contexts_.emplace_back(
// dropping directly, without delegating to this adapter. parent, std::move(encoder_context),
encoder->RegisterEncodeCompleteCallback(encoded_complete_callback_);
encoder_contexts_.emplace_back(
/*parent=*/nullptr, std::move(encoder),
/*framerate_controller=*/nullptr, /*stream_idx=*/0,
stream_codec.width, stream_codec.height, send_stream);
break;
}
encoder_contexts_.emplace_back(
this, std::move(encoder),
std::make_unique<FramerateController>(stream_codec.maxFramerate), std::make_unique<FramerateController>(stream_codec.maxFramerate),
/*stream_idx=*/i, stream_codec.width, stream_codec.height, send_stream); stream_idx, stream_codec.width, stream_codec.height, is_paused);
} }
// To save memory, don't store encoders that we don't use. // To save memory, don't store encoders that we don't use.
DestroyStoredEncoders(); DestroyStoredEncoders();
rtc::AtomicOps::ReleaseStore(&inited_, 1); rtc::AtomicOps::ReleaseStore(&inited_, 1);
return WEBRTC_VIDEO_CODEC_OK; return WEBRTC_VIDEO_CODEC_OK;
} }
@ -436,7 +448,7 @@ int SimulcastEncoderAdapter::Encode(
return WEBRTC_VIDEO_CODEC_ERROR; return WEBRTC_VIDEO_CODEC_ERROR;
} }
if (encoder_info_override_.apply_alignment_to_all_simulcast_layers()) { if (encoder_info_override_.apply_alignment_to_all_simulcast_layers()) {
for (const auto& layer : encoder_contexts_) { for (const auto& layer : stream_contexts_) {
if (layer.width() % alignment != 0 || layer.height() % alignment != 0) { if (layer.width() % alignment != 0 || layer.height() % alignment != 0) {
RTC_LOG(LS_WARNING) RTC_LOG(LS_WARNING)
<< "Codec " << layer.width() << "x" << layer.height() << "Codec " << layer.width() << "x" << layer.height()
@ -449,20 +461,22 @@ int SimulcastEncoderAdapter::Encode(
// All active streams should generate a key frame if // All active streams should generate a key frame if
// a key frame is requested by any stream. // a key frame is requested by any stream.
bool send_key_frame = false; bool is_keyframe_needed = false;
if (frame_types) { if (frame_types) {
for (size_t i = 0; i < frame_types->size(); ++i) { for (const auto& frame_type : *frame_types) {
if (frame_types->at(i) == VideoFrameType::kVideoFrameKey) { if (frame_type == VideoFrameType::kVideoFrameKey) {
send_key_frame = true; is_keyframe_needed = true;
break; break;
} }
} }
} }
for (const auto& layer : encoder_contexts_) { if (!is_keyframe_needed) {
if (layer.needs_keyframe()) { for (const auto& layer : stream_contexts_) {
send_key_frame = true; if (layer.is_keyframe_needed()) {
break; is_keyframe_needed = true;
break;
}
} }
} }
@ -471,9 +485,9 @@ int SimulcastEncoderAdapter::Encode(
int src_width = input_image.width(); int src_width = input_image.width();
int src_height = input_image.height(); int src_height = input_image.height();
for (auto& layer : encoder_contexts_) { for (auto& layer : stream_contexts_) {
// Don't encode frames in resolutions that we don't intend to send. // Don't encode frames in resolutions that we don't intend to send.
if (!layer.send_stream()) { if (layer.is_paused()) {
continue; continue;
} }
@ -485,8 +499,8 @@ int SimulcastEncoderAdapter::Encode(
// frame types for all streams should be passed to the encoder unchanged. // frame types for all streams should be passed to the encoder unchanged.
// Otherwise a single per-encoder frame type is passed. // Otherwise a single per-encoder frame type is passed.
std::vector<VideoFrameType> stream_frame_types( std::vector<VideoFrameType> stream_frame_types(
encoder_contexts_.size() == 1 ? NumberOfStreams(codec_) : 1); bypass_mode_ ? total_streams_count_ : 1);
if (send_key_frame) { if (is_keyframe_needed) {
std::fill(stream_frame_types.begin(), stream_frame_types.end(), std::fill(stream_frame_types.begin(), stream_frame_types.end(),
VideoFrameType::kVideoFrameKey); VideoFrameType::kVideoFrameKey);
layer.OnKeyframe(frame_timestamp); layer.OnKeyframe(frame_timestamp);
@ -548,9 +562,10 @@ int SimulcastEncoderAdapter::RegisterEncodeCompleteCallback(
EncodedImageCallback* callback) { EncodedImageCallback* callback) {
RTC_DCHECK_RUN_ON(&encoder_queue_); RTC_DCHECK_RUN_ON(&encoder_queue_);
encoded_complete_callback_ = callback; encoded_complete_callback_ = callback;
if (encoder_contexts_.size() == 1) { if (!stream_contexts_.empty() && stream_contexts_.front().stream_idx() == 0) {
encoder_contexts_.front().encoder().RegisterEncodeCompleteCallback( // Bypass frame encode complete callback for the lowest layer since there is
callback); // no need to override frame's spatial index.
stream_contexts_.front().encoder().RegisterEncodeCompleteCallback(callback);
} }
return WEBRTC_VIDEO_CODEC_OK; return WEBRTC_VIDEO_CODEC_OK;
} }
@ -571,31 +586,21 @@ void SimulcastEncoderAdapter::SetRates(
codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps + 0.5); codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
if (encoder_contexts_.size() == 1) { if (bypass_mode_) {
// Not doing simulcast. stream_contexts_.front().encoder().SetRates(parameters);
encoder_contexts_.front().encoder().SetRates(parameters);
return; return;
} }
num_active_streams_ = 0; for (StreamContext& layer_context : stream_contexts_) {
first_active_stream_idx_ = 0; int stream_idx = layer_context.stream_idx();
for (size_t stream_idx = 0; stream_idx < encoder_contexts_.size();
++stream_idx) {
EncoderContext& layer = encoder_contexts_[stream_idx];
uint32_t stream_bitrate_kbps = uint32_t stream_bitrate_kbps =
parameters.bitrate.GetSpatialLayerSum(stream_idx) / 1000; parameters.bitrate.GetSpatialLayerSum(stream_idx) / 1000;
if (stream_bitrate_kbps > 0) {
if (num_active_streams_ == 0) {
first_active_stream_idx_ = stream_idx;
}
++num_active_streams_;
}
// Need a key frame if we have not sent this stream before. // Need a key frame if we have not sent this stream before.
if (stream_bitrate_kbps > 0 && !layer.send_stream()) { if (stream_bitrate_kbps > 0 && layer_context.is_paused()) {
layer.set_keyframe_needed(); layer_context.set_is_keyframe_needed();
} }
layer.set_send_stream(stream_bitrate_kbps > 0); layer_context.set_is_paused(stream_bitrate_kbps == 0);
// Slice the temporal layers out of the full allocation and pass it on to // Slice the temporal layers out of the full allocation and pass it on to
// the encoder handling the current simulcast stream. // the encoder handling the current simulcast stream.
@ -623,29 +628,29 @@ void SimulcastEncoderAdapter::SetRates(
} }
} }
stream_parameters.framerate_fps = stream_parameters.framerate_fps = std::min<double>(
std::min<double>(parameters.framerate_fps, parameters.framerate_fps,
layer.target_fps().value_or(parameters.framerate_fps)); layer_context.target_fps().value_or(parameters.framerate_fps));
layer.encoder().SetRates(stream_parameters); layer_context.encoder().SetRates(stream_parameters);
} }
} }
void SimulcastEncoderAdapter::OnPacketLossRateUpdate(float packet_loss_rate) { void SimulcastEncoderAdapter::OnPacketLossRateUpdate(float packet_loss_rate) {
for (auto& c : encoder_contexts_) { for (auto& c : stream_contexts_) {
c.encoder().OnPacketLossRateUpdate(packet_loss_rate); c.encoder().OnPacketLossRateUpdate(packet_loss_rate);
} }
} }
void SimulcastEncoderAdapter::OnRttUpdate(int64_t rtt_ms) { void SimulcastEncoderAdapter::OnRttUpdate(int64_t rtt_ms) {
for (auto& c : encoder_contexts_) { for (auto& c : stream_contexts_) {
c.encoder().OnRttUpdate(rtt_ms); c.encoder().OnRttUpdate(rtt_ms);
} }
} }
void SimulcastEncoderAdapter::OnLossNotification( void SimulcastEncoderAdapter::OnLossNotification(
const LossNotification& loss_notification) { const LossNotification& loss_notification) {
for (auto& c : encoder_contexts_) { for (auto& c : stream_contexts_) {
c.encoder().OnLossNotification(loss_notification); c.encoder().OnLossNotification(loss_notification);
} }
} }
@ -669,72 +674,114 @@ void SimulcastEncoderAdapter::OnDroppedFrame(size_t stream_idx) {
// Not yet implemented. // Not yet implemented.
} }
void SimulcastEncoderAdapter::PopulateStreamCodec(
const webrtc::VideoCodec& inst,
int stream_index,
uint32_t start_bitrate_kbps,
StreamResolution stream_resolution,
webrtc::VideoCodec* stream_codec) {
*stream_codec = inst;
// Stream specific simulcast settings.
const SpatialLayer* spatial_layers = inst.simulcastStream;
stream_codec->numberOfSimulcastStreams = 0;
stream_codec->width = spatial_layers[stream_index].width;
stream_codec->height = spatial_layers[stream_index].height;
stream_codec->maxBitrate = spatial_layers[stream_index].maxBitrate;
stream_codec->minBitrate = spatial_layers[stream_index].minBitrate;
stream_codec->maxFramerate = spatial_layers[stream_index].maxFramerate;
stream_codec->qpMax = spatial_layers[stream_index].qpMax;
stream_codec->active = spatial_layers[stream_index].active;
// Settings that are based on stream/resolution.
if (stream_resolution == StreamResolution::LOWEST) {
// Settings for lowest spatial resolutions.
if (inst.mode == VideoCodecMode::kScreensharing) {
if (experimental_boosted_screenshare_qp_) {
stream_codec->qpMax = *experimental_boosted_screenshare_qp_;
}
} else if (boost_base_layer_quality_) {
stream_codec->qpMax = kLowestResMaxQp;
}
}
if (inst.codecType == webrtc::kVideoCodecVP8) {
stream_codec->VP8()->numberOfTemporalLayers =
spatial_layers[stream_index].numberOfTemporalLayers;
if (stream_resolution != StreamResolution::HIGHEST) {
// For resolutions below CIF, set the codec |complexity| parameter to
// kComplexityHigher, which maps to cpu_used = -4.
int pixels_per_frame = stream_codec->width * stream_codec->height;
if (pixels_per_frame < 352 * 288) {
stream_codec->VP8()->complexity =
webrtc::VideoCodecComplexity::kComplexityHigher;
}
// Turn off denoising for all streams but the highest resolution.
stream_codec->VP8()->denoisingOn = false;
}
} else if (inst.codecType == webrtc::kVideoCodecH264) {
stream_codec->H264()->numberOfTemporalLayers =
spatial_layers[stream_index].numberOfTemporalLayers;
}
stream_codec->startBitrate = start_bitrate_kbps;
// Legacy screenshare mode is only enabled for the first simulcast layer
stream_codec->legacy_conference_mode =
inst.legacy_conference_mode && stream_index == 0;
}
bool SimulcastEncoderAdapter::Initialized() const { bool SimulcastEncoderAdapter::Initialized() const {
return rtc::AtomicOps::AcquireLoad(&inited_) == 1; return rtc::AtomicOps::AcquireLoad(&inited_) == 1;
} }
void SimulcastEncoderAdapter::DestroyStoredEncoders() { void SimulcastEncoderAdapter::DestroyStoredEncoders() {
while (!stored_encoders_.empty()) { while (!cached_encoder_contexts_.empty()) {
stored_encoders_.pop(); cached_encoder_contexts_.pop_back();
} }
} }
std::unique_ptr<SimulcastEncoderAdapter::EncoderContext>
SimulcastEncoderAdapter::FetchOrCreateEncoderContext(
bool is_lowest_quality_stream) {
bool prefer_temporal_support = fallback_encoder_factory_ != nullptr &&
is_lowest_quality_stream &&
prefer_temporal_support_on_base_layer_;
// Toggling of |prefer_temporal_support| requires encoder recreation. Find
// and reuse encoder with desired |prefer_temporal_support|. Otherwise, if
// there is no such encoder in the cache, create a new instance.
auto encoder_context_iter =
std::find_if(cached_encoder_contexts_.begin(),
cached_encoder_contexts_.end(), [&](auto& encoder_context) {
return encoder_context->prefer_temporal_support() ==
prefer_temporal_support;
});
std::unique_ptr<SimulcastEncoderAdapter::EncoderContext> encoder_context;
if (encoder_context_iter != cached_encoder_contexts_.end()) {
encoder_context = std::move(*encoder_context_iter);
cached_encoder_contexts_.erase(encoder_context_iter);
} else {
std::unique_ptr<VideoEncoder> encoder =
primary_encoder_factory_->CreateVideoEncoder(video_format_);
if (fallback_encoder_factory_ != nullptr) {
encoder = CreateVideoEncoderSoftwareFallbackWrapper(
fallback_encoder_factory_->CreateVideoEncoder(video_format_),
std::move(encoder), prefer_temporal_support);
}
encoder_context = std::make_unique<SimulcastEncoderAdapter::EncoderContext>(
std::move(encoder), prefer_temporal_support);
}
encoder_context->encoder().RegisterEncodeCompleteCallback(
encoded_complete_callback_);
return encoder_context;
}
webrtc::VideoCodec SimulcastEncoderAdapter::MakeStreamCodec(
const webrtc::VideoCodec& codec,
int stream_idx,
uint32_t start_bitrate_kbps,
bool is_lowest_quality_stream,
bool is_highest_quality_stream) {
webrtc::VideoCodec codec_params = codec;
const SpatialLayer& stream_params = codec.simulcastStream[stream_idx];
codec_params.numberOfSimulcastStreams = 0;
codec_params.width = stream_params.width;
codec_params.height = stream_params.height;
codec_params.maxBitrate = stream_params.maxBitrate;
codec_params.minBitrate = stream_params.minBitrate;
codec_params.maxFramerate = stream_params.maxFramerate;
codec_params.qpMax = stream_params.qpMax;
codec_params.active = stream_params.active;
// Settings that are based on stream/resolution.
if (is_lowest_quality_stream) {
// Settings for lowest spatial resolutions.
if (codec.mode == VideoCodecMode::kScreensharing) {
if (experimental_boosted_screenshare_qp_) {
codec_params.qpMax = *experimental_boosted_screenshare_qp_;
}
} else if (boost_base_layer_quality_) {
codec_params.qpMax = kLowestResMaxQp;
}
}
if (codec.codecType == webrtc::kVideoCodecVP8) {
codec_params.VP8()->numberOfTemporalLayers =
stream_params.numberOfTemporalLayers;
if (!is_highest_quality_stream) {
// For resolutions below CIF, set the codec |complexity| parameter to
// kComplexityHigher, which maps to cpu_used = -4.
int pixels_per_frame = codec_params.width * codec_params.height;
if (pixels_per_frame < 352 * 288) {
codec_params.VP8()->complexity =
webrtc::VideoCodecComplexity::kComplexityHigher;
}
// Turn off denoising for all streams but the highest resolution.
codec_params.VP8()->denoisingOn = false;
}
} else if (codec.codecType == webrtc::kVideoCodecH264) {
codec_params.H264()->numberOfTemporalLayers =
stream_params.numberOfTemporalLayers;
}
// Cap start bitrate to the min bitrate in order to avoid strange codec
// behavior.
codec_params.startBitrate =
std::max(stream_params.minBitrate, start_bitrate_kbps);
// Legacy screenshare mode is only enabled for the first simulcast layer
codec_params.legacy_conference_mode =
codec.legacy_conference_mode && stream_idx == 0;
return codec_params;
}
void SimulcastEncoderAdapter::OverrideFromFieldTrial( void SimulcastEncoderAdapter::OverrideFromFieldTrial(
VideoEncoder::EncoderInfo* info) const { VideoEncoder::EncoderInfo* info) const {
if (encoder_info_override_.requested_resolution_alignment()) { if (encoder_info_override_.requested_resolution_alignment()) {
@ -750,10 +797,10 @@ void SimulcastEncoderAdapter::OverrideFromFieldTrial(
} }
VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const { VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const {
if (encoder_contexts_.size() == 1) { if (stream_contexts_.size() == 1) {
// Not using simulcast adapting functionality, just pass through. // Not using simulcast adapting functionality, just pass through.
VideoEncoder::EncoderInfo info = VideoEncoder::EncoderInfo info =
encoder_contexts_.front().encoder().GetEncoderInfo(); stream_contexts_.front().encoder().GetEncoderInfo();
OverrideFromFieldTrial(&info); OverrideFromFieldTrial(&info);
return info; return info;
} }
@ -764,17 +811,16 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const {
encoder_info.apply_alignment_to_all_simulcast_layers = false; encoder_info.apply_alignment_to_all_simulcast_layers = false;
encoder_info.supports_native_handle = true; encoder_info.supports_native_handle = true;
encoder_info.scaling_settings.thresholds = absl::nullopt; encoder_info.scaling_settings.thresholds = absl::nullopt;
if (encoder_contexts_.empty()) { if (stream_contexts_.empty()) {
OverrideFromFieldTrial(&encoder_info); OverrideFromFieldTrial(&encoder_info);
return encoder_info; return encoder_info;
} }
encoder_info.scaling_settings = VideoEncoder::ScalingSettings::kOff; encoder_info.scaling_settings = VideoEncoder::ScalingSettings::kOff;
auto active_streams = ActiveStreams(codec_);
for (size_t i = 0; i < encoder_contexts_.size(); ++i) { for (size_t i = 0; i < stream_contexts_.size(); ++i) {
VideoEncoder::EncoderInfo encoder_impl_info = VideoEncoder::EncoderInfo encoder_impl_info =
encoder_contexts_[i].encoder().GetEncoderInfo(); stream_contexts_[i].encoder().GetEncoderInfo();
if (i == 0) { if (i == 0) {
// Encoder name indicates names of all sub-encoders. // Encoder name indicates names of all sub-encoders.
@ -817,10 +863,6 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const {
if (encoder_impl_info.apply_alignment_to_all_simulcast_layers) { if (encoder_impl_info.apply_alignment_to_all_simulcast_layers) {
encoder_info.apply_alignment_to_all_simulcast_layers = true; encoder_info.apply_alignment_to_all_simulcast_layers = true;
} }
if (active_streams.num_active_streams == 1 &&
codec_.simulcastStream[i].active) {
encoder_info.scaling_settings = encoder_impl_info.scaling_settings;
}
} }
encoder_info.implementation_name += ")"; encoder_info.implementation_name += ")";

89
media/engine/simulcast_encoder_adapter.h
View file

@ -12,6 +12,7 @@
#ifndef MEDIA_ENGINE_SIMULCAST_ENCODER_ADAPTER_H_ #ifndef MEDIA_ENGINE_SIMULCAST_ENCODER_ADAPTER_H_
#define MEDIA_ENGINE_SIMULCAST_ENCODER_ADAPTER_H_ #define MEDIA_ENGINE_SIMULCAST_ENCODER_ADAPTER_H_
#include <list>
#include <memory> #include <memory>
#include <stack> #include <stack>
#include <string> #include <string>
@ -67,32 +68,50 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder {
EncoderInfo GetEncoderInfo() const override; EncoderInfo GetEncoderInfo() const override;
private: private:
class EncoderContext : public EncodedImageCallback { class EncoderContext {
public: public:
EncoderContext(SimulcastEncoderAdapter* parent, EncoderContext(std::unique_ptr<VideoEncoder> encoder,
std::unique_ptr<VideoEncoder> encoder, bool prefer_temporal_support);
std::unique_ptr<FramerateController> framerate_controller,
int stream_idx,
uint16_t width,
uint16_t height,
bool send_stream);
EncoderContext(EncoderContext&& rhs);
EncoderContext& operator=(EncoderContext&&) = delete; EncoderContext& operator=(EncoderContext&&) = delete;
~EncoderContext() override;
VideoEncoder& encoder() { return *encoder_; }
bool prefer_temporal_support() { return prefer_temporal_support_; }
void Release();
private:
std::unique_ptr<VideoEncoder> encoder_;
bool prefer_temporal_support_;
};
class StreamContext : public EncodedImageCallback {
public:
StreamContext(SimulcastEncoderAdapter* parent,
std::unique_ptr<EncoderContext> encoder_context,
std::unique_ptr<FramerateController> framerate_controller,
int stream_idx,
uint16_t width,
uint16_t height,
bool send_stream);
StreamContext(StreamContext&& rhs);
StreamContext& operator=(StreamContext&&) = delete;
~StreamContext() override;
Result OnEncodedImage( Result OnEncodedImage(
const EncodedImage& encoded_image, const EncodedImage& encoded_image,
const CodecSpecificInfo* codec_specific_info) override; const CodecSpecificInfo* codec_specific_info) override;
void OnDroppedFrame(DropReason reason) override; void OnDroppedFrame(DropReason reason) override;
VideoEncoder& encoder() { return *encoder_; } VideoEncoder& encoder() { return encoder_context_->encoder(); }
const VideoEncoder& encoder() const { return *encoder_; } const VideoEncoder& encoder() const { return encoder_context_->encoder(); }
int stream_idx() const { return stream_idx_; }
uint16_t width() const { return width_; } uint16_t width() const { return width_; }
uint16_t height() const { return height_; } uint16_t height() const { return height_; }
bool needs_keyframe() const { return send_stream_ && needs_keyframe_; } bool is_keyframe_needed() const {
void set_keyframe_needed() { needs_keyframe_ = true; } return !is_paused_ && is_keyframe_needed_;
bool send_stream() const { return send_stream_; } }
void set_send_stream(bool send_stream) { send_stream_ = send_stream; } void set_is_keyframe_needed() { is_keyframe_needed_ = true; }
bool is_paused() const { return is_paused_; }
void set_is_paused(bool is_paused) { is_paused_ = is_paused; }
absl::optional<float> target_fps() const { absl::optional<float> target_fps() const {
return framerate_controller_ == nullptr return framerate_controller_ == nullptr
? absl::nullopt ? absl::nullopt
@ -100,38 +119,34 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder {
framerate_controller_->GetTargetRate()); framerate_controller_->GetTargetRate());
} }
std::unique_ptr<VideoEncoder> Release() &&; std::unique_ptr<EncoderContext> ReleaseEncoderContext() &&;
void OnKeyframe(Timestamp timestamp); void OnKeyframe(Timestamp timestamp);
bool ShouldDropFrame(Timestamp timestamp); bool ShouldDropFrame(Timestamp timestamp);
private: private:
SimulcastEncoderAdapter* const parent_; SimulcastEncoderAdapter* const parent_;
std::unique_ptr<VideoEncoder> encoder_; std::unique_ptr<EncoderContext> encoder_context_;
std::unique_ptr<FramerateController> framerate_controller_; std::unique_ptr<FramerateController> framerate_controller_;
const int stream_idx_; const int stream_idx_;
const uint16_t width_; const uint16_t width_;
const uint16_t height_; const uint16_t height_;
bool needs_keyframe_; bool is_keyframe_needed_;
bool send_stream_; bool is_paused_;
}; };
enum class StreamResolution {
OTHER,
HIGHEST,
LOWEST,
};
// Populate the codec settings for each simulcast stream.
void PopulateStreamCodec(const webrtc::VideoCodec& inst,
int stream_index,
uint32_t start_bitrate_kbps,
StreamResolution stream_resolution,
webrtc::VideoCodec* stream_codec);
bool Initialized() const; bool Initialized() const;
void DestroyStoredEncoders(); void DestroyStoredEncoders();
std::unique_ptr<EncoderContext> FetchOrCreateEncoderContext(
bool is_lowest_quality_stream);
webrtc::VideoCodec MakeStreamCodec(const webrtc::VideoCodec& codec,
int stream_idx,
uint32_t start_bitrate_kbps,
bool is_lowest_quality_stream,
bool is_highest_quality_stream);
EncodedImageCallback::Result OnEncodedImage( EncodedImageCallback::Result OnEncodedImage(
size_t stream_idx, size_t stream_idx,
const EncodedImage& encoded_image, const EncodedImage& encoded_image,
@ -146,17 +161,17 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder {
VideoEncoderFactory* const fallback_encoder_factory_; VideoEncoderFactory* const fallback_encoder_factory_;
const SdpVideoFormat video_format_; const SdpVideoFormat video_format_;
VideoCodec codec_; VideoCodec codec_;
std::vector<EncoderContext> encoder_contexts_; int total_streams_count_;
bool bypass_mode_;
std::vector<StreamContext> stream_contexts_;
EncodedImageCallback* encoded_complete_callback_; EncodedImageCallback* encoded_complete_callback_;
size_t first_active_stream_idx_;
size_t num_active_streams_;
// Used for checking the single-threaded access of the encoder interface. // Used for checking the single-threaded access of the encoder interface.
RTC_NO_UNIQUE_ADDRESS SequenceChecker encoder_queue_; RTC_NO_UNIQUE_ADDRESS SequenceChecker encoder_queue_;
// Store encoders in between calls to Release and InitEncode, so they don't // Store encoders in between calls to Release and InitEncode, so they don't
// have to be recreated. Remaining encoders are destroyed by the destructor. // have to be recreated. Remaining encoders are destroyed by the destructor.
std::stack<std::unique_ptr<VideoEncoder>> stored_encoders_; std::list<std::unique_ptr<EncoderContext>> cached_encoder_contexts_;
const absl::optional<unsigned int> experimental_boosted_screenshare_qp_; const absl::optional<unsigned int> experimental_boosted_screenshare_qp_;
const bool boost_base_layer_quality_; const bool boost_base_layer_quality_;

102
media/engine/simulcast_encoder_adapter_unittest.cc
View file

@ -18,6 +18,7 @@
#include "api/test/simulcast_test_fixture.h" #include "api/test/simulcast_test_fixture.h"
#include "api/test/video/function_video_decoder_factory.h" #include "api/test/video/function_video_decoder_factory.h"
#include "api/test/video/function_video_encoder_factory.h" #include "api/test/video/function_video_encoder_factory.h"
#include "api/video/video_codec_constants.h"
#include "api/video_codecs/sdp_video_format.h" #include "api/video_codecs/sdp_video_format.h"
#include "api/video_codecs/video_encoder.h" #include "api/video_codecs/video_encoder.h"
#include "api/video_codecs/video_encoder_factory.h" #include "api/video_codecs/video_encoder_factory.h"
@ -421,14 +422,24 @@ class TestSimulcastEncoderAdapterFake : public ::testing::Test,
} }
void SetUp() override { void SetUp() override {
helper_ = std::make_unique<TestSimulcastEncoderAdapterFakeHelper>( helper_.reset(new TestSimulcastEncoderAdapterFakeHelper(
use_fallback_factory_, SdpVideoFormat("VP8", sdp_video_parameters_)); use_fallback_factory_, SdpVideoFormat("VP8", sdp_video_parameters_)));
adapter_.reset(helper_->CreateMockEncoderAdapter()); adapter_.reset(helper_->CreateMockEncoderAdapter());
last_encoded_image_width_ = -1; last_encoded_image_width_ = -1;
last_encoded_image_height_ = -1; last_encoded_image_height_ = -1;
last_encoded_image_simulcast_index_ = -1; last_encoded_image_simulcast_index_ = -1;
} }
void ReSetUp() {
if (adapter_) {
adapter_->Release();
// |helper_| owns factories which |adapter_| needs to destroy encoders.
// Release |adapter_| before |helper_| (released in SetUp()).
adapter_.reset();
}
SetUp();
}
Result OnEncodedImage(const EncodedImage& encoded_image, Result OnEncodedImage(const EncodedImage& encoded_image,
const CodecSpecificInfo* codec_specific_info) override { const CodecSpecificInfo* codec_specific_info) override {
last_encoded_image_width_ = encoded_image._encodedWidth; last_encoded_image_width_ = encoded_image._encodedWidth;
@ -451,10 +462,23 @@ class TestSimulcastEncoderAdapterFake : public ::testing::Test,
return true; return true;
} }
void SetupCodec() { void SetupCodec() { SetupCodec(/*active_streams=*/{true, true, true}); }
void SetupCodec(std::vector<bool> active_streams) {
SimulcastTestFixtureImpl::DefaultSettings( SimulcastTestFixtureImpl::DefaultSettings(
&codec_, static_cast<const int*>(kTestTemporalLayerProfile), &codec_, static_cast<const int*>(kTestTemporalLayerProfile),
kVideoCodecVP8); kVideoCodecVP8);
ASSERT_LE(active_streams.size(), codec_.numberOfSimulcastStreams);
codec_.numberOfSimulcastStreams = active_streams.size();
for (size_t stream_idx = 0; stream_idx < kMaxSimulcastStreams;
++stream_idx) {
if (stream_idx >= codec_.numberOfSimulcastStreams) {
// Reset parameters of unspecified stream.
codec_.simulcastStream[stream_idx] = {0};
} else {
codec_.simulcastStream[stream_idx].active = active_streams[stream_idx];
}
}
rate_allocator_.reset(new SimulcastRateAllocator(codec_)); rate_allocator_.reset(new SimulcastRateAllocator(codec_));
EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings));
adapter_->RegisterEncodeCompleteCallback(this); adapter_->RegisterEncodeCompleteCallback(this);
@ -579,7 +603,8 @@ TEST_F(TestSimulcastEncoderAdapterFake, EncodedCallbackForDifferentEncoders) {
EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index));
EXPECT_EQ(1152, width); EXPECT_EQ(1152, width);
EXPECT_EQ(704, height); EXPECT_EQ(704, height);
EXPECT_EQ(0, simulcast_index); // SEA doesn't intercept frame encode complete callback for the lowest stream.
EXPECT_EQ(-1, simulcast_index);
encoders[1]->SendEncodedImage(300, 620); encoders[1]->SendEncodedImage(300, 620);
EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index));
@ -795,7 +820,8 @@ TEST_F(TestSimulcastEncoderAdapterFake, ReinitDoesNotReorderFrameSimulcastIdx) {
int height; int height;
int simulcast_index; int simulcast_index;
EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index));
EXPECT_EQ(0, simulcast_index); // SEA doesn't intercept frame encode complete callback for the lowest stream.
EXPECT_EQ(-1, simulcast_index);
encoders[1]->SendEncodedImage(300, 620); encoders[1]->SendEncodedImage(300, 620);
EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index));
@ -815,7 +841,7 @@ TEST_F(TestSimulcastEncoderAdapterFake, ReinitDoesNotReorderFrameSimulcastIdx) {
// Verify that the same encoder sends out frames on the same simulcast index. // Verify that the same encoder sends out frames on the same simulcast index.
encoders[0]->SendEncodedImage(1152, 704); encoders[0]->SendEncodedImage(1152, 704);
EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index));
EXPECT_EQ(0, simulcast_index); EXPECT_EQ(-1, simulcast_index);
encoders[1]->SendEncodedImage(300, 620); encoders[1]->SendEncodedImage(300, 620);
EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index)); EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index));
@ -1593,5 +1619,69 @@ TEST_F(TestSimulcastEncoderAdapterFake, SupportsPerSimulcastLayerMaxFramerate) {
EXPECT_EQ(10u, helper_->factory()->encoders()[2]->codec().maxFramerate); EXPECT_EQ(10u, helper_->factory()->encoders()[2]->codec().maxFramerate);
} }
TEST_F(TestSimulcastEncoderAdapterFake, CreatesEncoderOnlyIfStreamIsActive) {
// Legacy singlecast
SetupCodec(/*active_streams=*/{});
EXPECT_EQ(1u, helper_->factory()->encoders().size());
// Simulcast-capable underlaying encoder
ReSetUp();
helper_->factory()->set_supports_simulcast(true);
SetupCodec(/*active_streams=*/{true, true});
EXPECT_EQ(1u, helper_->factory()->encoders().size());
// Muti-encoder simulcast
ReSetUp();
helper_->factory()->set_supports_simulcast(false);
SetupCodec(/*active_streams=*/{true, true});
EXPECT_EQ(2u, helper_->factory()->encoders().size());
// Singlecast via layers deactivation. Lowest layer is active.
ReSetUp();
helper_->factory()->set_supports_simulcast(false);
SetupCodec(/*active_streams=*/{true, false});
EXPECT_EQ(1u, helper_->factory()->encoders().size());
// Singlecast via layers deactivation. Highest layer is active.
ReSetUp();
helper_->factory()->set_supports_simulcast(false);
SetupCodec(/*active_streams=*/{false, true});
EXPECT_EQ(1u, helper_->factory()->encoders().size());
}
TEST_F(TestSimulcastEncoderAdapterFake,
RecreateEncoderIfPreferTemporalSupportIsEnabled) {
// Normally SEA reuses encoders. But, when TL-based SW fallback is enabled,
// the encoder which served the lowest stream should be recreated before it
// can be used to process an upper layer and vice-versa.
test::ScopedFieldTrials field_trials(
"WebRTC-Video-PreferTemporalSupportOnBaseLayer/Enabled/");
use_fallback_factory_ = true;
ReSetUp();
// Legacy singlecast
SetupCodec(/*active_streams=*/{});
ASSERT_EQ(1u, helper_->factory()->encoders().size());
// Singlecast, the lowest stream is active. Encoder should be reused.
MockVideoEncoder* prev_encoder = helper_->factory()->encoders()[0];
SetupCodec(/*active_streams=*/{true, false});
ASSERT_EQ(1u, helper_->factory()->encoders().size());
EXPECT_EQ(helper_->factory()->encoders()[0], prev_encoder);
// Singlecast, an upper stream is active. Encoder should be recreated.
EXPECT_CALL(*prev_encoder, Release()).Times(1);
SetupCodec(/*active_streams=*/{false, true});
ASSERT_EQ(1u, helper_->factory()->encoders().size());
EXPECT_NE(helper_->factory()->encoders()[0], prev_encoder);
// Singlecast, the lowest stream is active. Encoder should be recreated.
prev_encoder = helper_->factory()->encoders()[0];
EXPECT_CALL(*prev_encoder, Release()).Times(1);
SetupCodec(/*active_streams=*/{true, false});
ASSERT_EQ(1u, helper_->factory()->encoders().size());
EXPECT_NE(helper_->factory()->encoders()[0], prev_encoder);
}
} // namespace test } // namespace test
} // namespace webrtc } // namespace webrtc

3
media/engine/webrtc_video_engine.cc
View file

@ -2306,6 +2306,9 @@ webrtc::RTCError WebRtcVideoChannel::WebRtcVideoSendStream::SetRtpParameters(
// TODO(bugs.webrtc.org/8807): The active field as well should not require // TODO(bugs.webrtc.org/8807): The active field as well should not require
// a full encoder reconfiguration, but it needs to update both the bitrate // a full encoder reconfiguration, but it needs to update both the bitrate
// allocator and the video bitrate allocator. // allocator and the video bitrate allocator.
//
// Note that the simulcast encoder adapter relies on the fact that layers
// de/activation triggers encoder reinitialization.
bool new_send_state = false; bool new_send_state = false;
for (size_t i = 0; i < rtp_parameters_.encodings.size(); ++i) { for (size_t i = 0; i < rtp_parameters_.encodings.size(); ++i) {
bool new_active = IsLayerActive(new_parameters.encodings[i]); bool new_active = IsLayerActive(new_parameters.encodings[i]);