From c98aebbbef26dfc4449d27d40307a945cedfec59 Mon Sep 17 00:00:00 2001 From: Ilya Nikolaevskiy Date: Tue, 13 Jul 2021 16:27:11 +0200 Subject: [PATCH] Change how alignment requirements are processed Software fallback wrapper now reports least common multiple of requirements for two encoders. SimulcastEncoderAdapter queries actual encoder before InitEncode call and requests alignment for all layers if simulcast is not supported by any of the encoders. Bug: chromium:1084702 Change-Id: Iaed8190737125d447036b6c664b863be72556a5d Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/225881 Reviewed-by: Niels Moller Commit-Queue: Ilya Nikolaevskiy Cr-Commit-Position: refs/heads/master@{#34466} --- ...video_encoder_software_fallback_wrapper.cc | 8 +++ media/engine/simulcast_encoder_adapter.cc | 63 ++++++++++++++++--- media/engine/simulcast_encoder_adapter.h | 23 +++++-- .../simulcast_encoder_adapter_unittest.cc | 4 +- 4 files changed, 81 insertions(+), 17 deletions(-) diff --git a/api/video_codecs/video_encoder_software_fallback_wrapper.cc b/api/video_codecs/video_encoder_software_fallback_wrapper.cc index be79c42464..bcce9dcd93 100644 --- a/api/video_codecs/video_encoder_software_fallback_wrapper.cc +++ b/api/video_codecs/video_encoder_software_fallback_wrapper.cc @@ -25,6 +25,7 @@ #include "api/video/video_frame.h" #include "api/video_codecs/video_codec.h" #include "api/video_codecs/video_encoder.h" +#include "media/base/video_common.h" #include "modules/video_coding/include/video_error_codes.h" #include "modules/video_coding/utility/simulcast_utility.h" #include "rtc_base/checks.h" @@ -417,6 +418,13 @@ VideoEncoder::EncoderInfo VideoEncoderSoftwareFallbackWrapper::GetEncoderInfo() EncoderInfo info = IsFallbackActive() ? fallback_encoder_info : default_encoder_info; + info.requested_resolution_alignment = cricket::LeastCommonMultiple( + fallback_encoder_info.requested_resolution_alignment, + default_encoder_info.requested_resolution_alignment); + info.apply_alignment_to_all_simulcast_layers = + fallback_encoder_info.apply_alignment_to_all_simulcast_layers || + default_encoder_info.apply_alignment_to_all_simulcast_layers; + if (fallback_params_.has_value()) { const auto settings = (encoder_state_ == EncoderState::kForcedFallback) ? fallback_encoder_info.scaling_settings diff --git a/media/engine/simulcast_encoder_adapter.cc b/media/engine/simulcast_encoder_adapter.cc index 3af022ab17..116f987aa4 100644 --- a/media/engine/simulcast_encoder_adapter.cc +++ b/media/engine/simulcast_encoder_adapter.cc @@ -149,9 +149,13 @@ namespace webrtc { SimulcastEncoderAdapter::EncoderContext::EncoderContext( std::unique_ptr encoder, - bool prefer_temporal_support) + bool prefer_temporal_support, + VideoEncoder::EncoderInfo primary_info, + VideoEncoder::EncoderInfo fallback_info) : encoder_(std::move(encoder)), - prefer_temporal_support_(prefer_temporal_support) {} + prefer_temporal_support_(prefer_temporal_support), + primary_info_(std::move(primary_info)), + fallback_info_(std::move(fallback_info)) {} void SimulcastEncoderAdapter::EncoderContext::Release() { if (encoder_) { @@ -690,7 +694,7 @@ void SimulcastEncoderAdapter::DestroyStoredEncoders() { std::unique_ptr SimulcastEncoderAdapter::FetchOrCreateEncoderContext( - bool is_lowest_quality_stream) { + bool is_lowest_quality_stream) const { bool prefer_temporal_support = fallback_encoder_factory_ != nullptr && is_lowest_quality_stream && prefer_temporal_support_on_base_layer_; @@ -712,14 +716,20 @@ SimulcastEncoderAdapter::FetchOrCreateEncoderContext( } else { std::unique_ptr encoder = primary_encoder_factory_->CreateVideoEncoder(video_format_); + VideoEncoder::EncoderInfo primary_info = encoder->GetEncoderInfo(); + VideoEncoder::EncoderInfo fallback_info = primary_info; if (fallback_encoder_factory_ != nullptr) { + std::unique_ptr fallback_encoder = + fallback_encoder_factory_->CreateVideoEncoder(video_format_); + fallback_info = fallback_encoder->GetEncoderInfo(); encoder = CreateVideoEncoderSoftwareFallbackWrapper( - fallback_encoder_factory_->CreateVideoEncoder(video_format_), - std::move(encoder), prefer_temporal_support); + std::move(fallback_encoder), std::move(encoder), + prefer_temporal_support); } encoder_context = std::make_unique( - std::move(encoder), prefer_temporal_support); + std::move(encoder), prefer_temporal_support, primary_info, + fallback_info); } encoder_context->encoder().RegisterEncodeCompleteCallback( @@ -789,9 +799,11 @@ webrtc::VideoCodec SimulcastEncoderAdapter::MakeStreamCodec( void SimulcastEncoderAdapter::OverrideFromFieldTrial( VideoEncoder::EncoderInfo* info) const { if (encoder_info_override_.requested_resolution_alignment()) { - info->requested_resolution_alignment = - *encoder_info_override_.requested_resolution_alignment(); + info->requested_resolution_alignment = cricket::LeastCommonMultiple( + info->requested_resolution_alignment, + *encoder_info_override_.requested_resolution_alignment()); info->apply_alignment_to_all_simulcast_layers = + info->apply_alignment_to_all_simulcast_layers || encoder_info_override_.apply_alignment_to_all_simulcast_layers(); } if (!encoder_info_override_.resolution_bitrate_limits().empty()) { @@ -815,7 +827,34 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const { encoder_info.apply_alignment_to_all_simulcast_layers = false; encoder_info.supports_native_handle = true; encoder_info.scaling_settings.thresholds = absl::nullopt; + if (stream_contexts_.empty()) { + // GetEncoderInfo queried before InitEncode. Only alignment info is needed + // to be filled. + // Create one encoder and query it. + + std::unique_ptr encoder_context = + FetchOrCreateEncoderContext(true); + + const VideoEncoder::EncoderInfo& primary_info = + encoder_context->PrimaryInfo(); + const VideoEncoder::EncoderInfo& fallback_info = + encoder_context->FallbackInfo(); + + encoder_info.requested_resolution_alignment = cricket::LeastCommonMultiple( + primary_info.requested_resolution_alignment, + fallback_info.requested_resolution_alignment); + + encoder_info.apply_alignment_to_all_simulcast_layers = + primary_info.apply_alignment_to_all_simulcast_layers || + fallback_info.apply_alignment_to_all_simulcast_layers; + + if (!primary_info.supports_simulcast || !fallback_info.supports_simulcast) { + encoder_info.apply_alignment_to_all_simulcast_layers = true; + } + + cached_encoder_contexts_.emplace_back(std::move(encoder_context)); + OverrideFromFieldTrial(&encoder_info); return encoder_info; } @@ -825,7 +864,6 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const { for (size_t i = 0; i < stream_contexts_.size(); ++i) { VideoEncoder::EncoderInfo encoder_impl_info = stream_contexts_[i].encoder().GetEncoderInfo(); - if (i == 0) { // Encoder name indicates names of all sub-encoders. encoder_info.implementation_name += " ("; @@ -864,7 +902,12 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const { encoder_info.requested_resolution_alignment = cricket::LeastCommonMultiple( encoder_info.requested_resolution_alignment, encoder_impl_info.requested_resolution_alignment); - if (encoder_impl_info.apply_alignment_to_all_simulcast_layers) { + // request alignment on all layers if any of the encoders may need it, or + // if any non-top layer encoder requests a non-trivial alignment. + if (encoder_impl_info.apply_alignment_to_all_simulcast_layers || + (encoder_impl_info.requested_resolution_alignment > 1 && + (codec_.simulcastStream[i].height < codec_.height || + codec_.simulcastStream[i].width < codec_.width))) { encoder_info.apply_alignment_to_all_simulcast_layers = true; } } diff --git a/media/engine/simulcast_encoder_adapter.h b/media/engine/simulcast_encoder_adapter.h index 2cb29edfd6..07e3ccd024 100644 --- a/media/engine/simulcast_encoder_adapter.h +++ b/media/engine/simulcast_encoder_adapter.h @@ -71,16 +71,24 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder { class EncoderContext { public: EncoderContext(std::unique_ptr encoder, - bool prefer_temporal_support); + bool prefer_temporal_support, + VideoEncoder::EncoderInfo primary_info, + VideoEncoder::EncoderInfo fallback_info); EncoderContext& operator=(EncoderContext&&) = delete; VideoEncoder& encoder() { return *encoder_; } bool prefer_temporal_support() { return prefer_temporal_support_; } void Release(); + const VideoEncoder::EncoderInfo& PrimaryInfo() { return primary_info_; } + + const VideoEncoder::EncoderInfo& FallbackInfo() { return fallback_info_; } + private: std::unique_ptr encoder_; bool prefer_temporal_support_; + const VideoEncoder::EncoderInfo primary_info_; + const VideoEncoder::EncoderInfo fallback_info_; }; class StreamContext : public EncodedImageCallback { @@ -138,8 +146,11 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder { void DestroyStoredEncoders(); + // This method creates encoder. May reuse previously created encoders from + // |cached_encoder_contexts_|. It's const because it's used from + // const GetEncoderInfo(). std::unique_ptr FetchOrCreateEncoderContext( - bool is_lowest_quality_stream); + bool is_lowest_quality_stream) const; webrtc::VideoCodec MakeStreamCodec(const webrtc::VideoCodec& codec, int stream_idx, @@ -169,9 +180,11 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder { // Used for checking the single-threaded access of the encoder interface. RTC_NO_UNIQUE_ADDRESS SequenceChecker encoder_queue_; - // Store encoders in between calls to Release and InitEncode, so they don't - // have to be recreated. Remaining encoders are destroyed by the destructor. - std::list> cached_encoder_contexts_; + // Store previously created and released encoders , so they don't have to be + // recreated. Remaining encoders are destroyed by the destructor. + // Marked as |mutable| becuase we may need to temporarily create encoder in + // GetEncoderInfo(), which is const. + mutable std::list> cached_encoder_contexts_; const absl::optional experimental_boosted_screenshare_qp_; const bool boost_base_layer_quality_; diff --git a/media/engine/simulcast_encoder_adapter_unittest.cc b/media/engine/simulcast_encoder_adapter_unittest.cc index a74a2c3785..48e005f1c2 100644 --- a/media/engine/simulcast_encoder_adapter_unittest.cc +++ b/media/engine/simulcast_encoder_adapter_unittest.cc @@ -900,8 +900,6 @@ TEST_F(TestSimulcastEncoderAdapterFake, SetRatesUnderMinBitrate) { } TEST_F(TestSimulcastEncoderAdapterFake, SupportsImplementationName) { - EXPECT_EQ("SimulcastEncoderAdapter", - adapter_->GetEncoderInfo().implementation_name); SimulcastTestFixtureImpl::DefaultSettings( &codec_, static_cast(kTestTemporalLayerProfile), kVideoCodecVP8); @@ -910,6 +908,8 @@ TEST_F(TestSimulcastEncoderAdapterFake, SupportsImplementationName) { encoder_names.push_back("codec2"); encoder_names.push_back("codec3"); helper_->factory()->SetEncoderNames(encoder_names); + EXPECT_EQ("SimulcastEncoderAdapter", + adapter_->GetEncoderInfo().implementation_name); EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings)); EXPECT_EQ("SimulcastEncoderAdapter (codec1, codec2, codec3)", adapter_->GetEncoderInfo().implementation_name);