diff --git a/api/video_codecs/BUILD.gn b/api/video_codecs/BUILD.gn index 8f35909e28..534a34e561 100644 --- a/api/video_codecs/BUILD.gn +++ b/api/video_codecs/BUILD.gn @@ -77,6 +77,7 @@ rtc_static_library("builtin_video_encoder_factory") { deps = [ ":video_codecs_api", + "../../media:rtc_encoder_simulcast_proxy", "../../media:rtc_internal_video_codecs", "../../media:rtc_media_base", "../../media:rtc_vp8_encoder_simulcast_proxy", diff --git a/api/video_codecs/builtin_video_encoder_factory.cc b/api/video_codecs/builtin_video_encoder_factory.cc index 9e382fa801..1778c218cb 100644 --- a/api/video_codecs/builtin_video_encoder_factory.cc +++ b/api/video_codecs/builtin_video_encoder_factory.cc @@ -18,8 +18,8 @@ #include "api/video_codecs/video_encoder.h" #include "media/base/codec.h" #include "media/base/mediaconstants.h" +#include "media/engine/encoder_simulcast_proxy.h" #include "media/engine/internalencoderfactory.h" -#include "media/engine/vp8_encoder_simulcast_proxy.h" #include "rtc_base/checks.h" namespace webrtc { @@ -63,7 +63,7 @@ class BuiltinVideoEncoderFactory : public VideoEncoderFactory { format)) { internal_encoder = absl::EqualsIgnoreCase(format.name, cricket::kVp8CodecName) - ? absl::make_unique( + ? absl::make_unique( internal_encoder_factory_.get(), format) : internal_encoder_factory_->CreateVideoEncoder(format); } diff --git a/media/BUILD.gn b/media/BUILD.gn index 4fa2d710c2..f54bb73a7c 100644 --- a/media/BUILD.gn +++ b/media/BUILD.gn @@ -214,12 +214,35 @@ rtc_static_library("rtc_vp8_encoder_simulcast_proxy") { } } +rtc_static_library("rtc_encoder_simulcast_proxy") { + visibility = [ "*" ] + defines = [] + libs = [] + sources = [ + "engine/encoder_simulcast_proxy.cc", + "engine/encoder_simulcast_proxy.h", + ] + deps = [ + ":rtc_simulcast_encoder_adapter", + "../:webrtc_common", + "../api/video:video_bitrate_allocation", + "../api/video:video_frame", + "../api/video_codecs:video_codecs_api", + "../modules/video_coding:video_codec_interface", + ] + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } +} + rtc_static_library("rtc_internal_video_codecs") { visibility = [ "*" ] allow_poison = [ "software_video_codecs" ] defines = [] libs = [] deps = [ + ":rtc_encoder_simulcast_proxy", ":rtc_h264_profile_id", ":rtc_simulcast_encoder_adapter", ":rtc_vp8_encoder_simulcast_proxy", @@ -244,7 +267,8 @@ rtc_static_library("rtc_internal_video_codecs") { "engine/scopedvideoencoder.h", # TODO(bugs.webrtc.org/7925): stop exporting this header once downstream - # targets depend on :rtc_vp8_encoder_simulcast_proxy directly. + # targets depend on :rtc_encoder_simulcast_proxy directly. + "engine/encoder_simulcast_proxy.h", "engine/vp8_encoder_simulcast_proxy.h", "engine/webrtcvideodecoderfactory.h", "engine/webrtcvideoencoderfactory.h", @@ -591,6 +615,7 @@ if (rtc_include_tests) { "base/videocapturer_unittest.cc", "base/videocommon_unittest.cc", "engine/apm_helpers_unittest.cc", + "engine/encoder_simulcast_proxy_unittest.cc", "engine/internaldecoderfactory_unittest.cc", "engine/multiplexcodecfactory_unittest.cc", "engine/nullwebrtcvideoengine_unittest.cc", @@ -643,6 +668,7 @@ if (rtc_include_tests) { } deps += [ + ":rtc_encoder_simulcast_proxy", ":rtc_internal_video_codecs", ":rtc_media", ":rtc_media_base", diff --git a/media/engine/convert_legacy_video_factory.cc b/media/engine/convert_legacy_video_factory.cc index cd60fa731b..4c30dc24a5 100644 --- a/media/engine/convert_legacy_video_factory.cc +++ b/media/engine/convert_legacy_video_factory.cc @@ -26,12 +26,12 @@ #include "api/video_codecs/video_encoder_software_fallback_wrapper.h" #include "media/base/codec.h" #include "media/base/mediaconstants.h" +#include "media/engine/encoder_simulcast_proxy.h" #include "media/engine/internaldecoderfactory.h" #include "media/engine/internalencoderfactory.h" #include "media/engine/scopedvideodecoder.h" #include "media/engine/scopedvideoencoder.h" #include "media/engine/simulcast_encoder_adapter.h" -#include "media/engine/vp8_encoder_simulcast_proxy.h" #include "media/engine/webrtcvideodecoderfactory.h" #include "media/engine/webrtcvideoencoderfactory.h" #include "rtc_base/checks.h" @@ -137,7 +137,7 @@ class EncoderAdapter : public webrtc::VideoEncoderFactory { format)) { internal_encoder = absl::EqualsIgnoreCase(format.name, kVp8CodecName) - ? absl::make_unique( + ? absl::make_unique( internal_encoder_factory_.get(), format) : internal_encoder_factory_->CreateVideoEncoder(format); } diff --git a/media/engine/encoder_simulcast_proxy.cc b/media/engine/encoder_simulcast_proxy.cc new file mode 100644 index 0000000000..3eccb6fae2 --- /dev/null +++ b/media/engine/encoder_simulcast_proxy.cc @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "media/engine/encoder_simulcast_proxy.h" + +#include "media/engine/simulcast_encoder_adapter.h" +#include "modules/video_coding/include/video_error_codes.h" + +namespace webrtc { +EncoderSimulcastProxy::EncoderSimulcastProxy(VideoEncoderFactory* factory, + const SdpVideoFormat& format) + : factory_(factory), video_format_(format), callback_(nullptr) { + encoder_ = factory_->CreateVideoEncoder(format); +} + +EncoderSimulcastProxy::EncoderSimulcastProxy(VideoEncoderFactory* factory) + : EncoderSimulcastProxy(factory, SdpVideoFormat("VP8")) {} + +EncoderSimulcastProxy::~EncoderSimulcastProxy() {} + +int EncoderSimulcastProxy::Release() { + return encoder_->Release(); +} + +int EncoderSimulcastProxy::InitEncode(const VideoCodec* inst, + int number_of_cores, + size_t max_payload_size) { + int ret = encoder_->InitEncode(inst, number_of_cores, max_payload_size); + if (ret == WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED) { + encoder_.reset(new SimulcastEncoderAdapter(factory_, video_format_)); + if (callback_) { + encoder_->RegisterEncodeCompleteCallback(callback_); + } + ret = encoder_->InitEncode(inst, number_of_cores, max_payload_size); + } + return ret; +} + +int EncoderSimulcastProxy::Encode(const VideoFrame& input_image, + const CodecSpecificInfo* codec_specific_info, + const std::vector* frame_types) { + return encoder_->Encode(input_image, codec_specific_info, frame_types); +} + +int EncoderSimulcastProxy::RegisterEncodeCompleteCallback( + EncodedImageCallback* callback) { + callback_ = callback; + return encoder_->RegisterEncodeCompleteCallback(callback); +} + +int EncoderSimulcastProxy::SetRateAllocation( + const VideoBitrateAllocation& bitrate, + uint32_t new_framerate) { + return encoder_->SetRateAllocation(bitrate, new_framerate); +} + +VideoEncoder::EncoderInfo EncoderSimulcastProxy::GetEncoderInfo() const { + return encoder_->GetEncoderInfo(); +} + +} // namespace webrtc diff --git a/media/engine/encoder_simulcast_proxy.h b/media/engine/encoder_simulcast_proxy.h new file mode 100644 index 0000000000..f01c5c36e7 --- /dev/null +++ b/media/engine/encoder_simulcast_proxy.h @@ -0,0 +1,65 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#ifndef MEDIA_ENGINE_ENCODER_SIMULCAST_PROXY_H_ +#define MEDIA_ENGINE_ENCODER_SIMULCAST_PROXY_H_ + +#include +#include +#include +#include + +#include "api/video/video_bitrate_allocation.h" +#include "api/video/video_frame.h" +#include "api/video_codecs/sdp_video_format.h" +#include "api/video_codecs/video_codec.h" +#include "api/video_codecs/video_encoder.h" +#include "api/video_codecs/video_encoder_factory.h" +#include "common_types.h" // NOLINT(build/include) +#include "modules/video_coding/include/video_codec_interface.h" + +namespace webrtc { + +// This class provides fallback to SimulcastEncoderAdapter if default VP8Encoder +// doesn't support simulcast for provided settings. +class EncoderSimulcastProxy : public VideoEncoder { + public: + EncoderSimulcastProxy(VideoEncoderFactory* factory, + const SdpVideoFormat& format); + // Deprecated. Remove once all clients use constructor with both factory and + // SdpVideoFormat; + explicit EncoderSimulcastProxy(VideoEncoderFactory* factory); + + ~EncoderSimulcastProxy() override; + + // Implements VideoEncoder. + int Release() override; + int InitEncode(const VideoCodec* inst, + int number_of_cores, + size_t max_payload_size) override; + int Encode(const VideoFrame& input_image, + const CodecSpecificInfo* codec_specific_info, + const std::vector* frame_types) override; + int RegisterEncodeCompleteCallback(EncodedImageCallback* callback) override; + int SetRateAllocation(const VideoBitrateAllocation& bitrate, + uint32_t new_framerate) override; + EncoderInfo GetEncoderInfo() const override; + + private: + VideoEncoderFactory* const factory_; + SdpVideoFormat video_format_; + std::unique_ptr encoder_; + EncodedImageCallback* callback_; +}; + +} // namespace webrtc + +#endif // MEDIA_ENGINE_ENCODER_SIMULCAST_PROXY_H_ diff --git a/media/engine/encoder_simulcast_proxy_unittest.cc b/media/engine/encoder_simulcast_proxy_unittest.cc new file mode 100644 index 0000000000..d1a88762b5 --- /dev/null +++ b/media/engine/encoder_simulcast_proxy_unittest.cc @@ -0,0 +1,242 @@ +/* + * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + * + */ + +#include "media/engine/encoder_simulcast_proxy.h" +#include + +#include "api/test/mock_video_encoder_factory.h" +#include "api/video_codecs/vp8_temporal_layers.h" +#include "media/engine/webrtcvideoencoderfactory.h" +#include "modules/video_coding/include/video_codec_interface.h" +#include "test/gmock.h" +#include "test/gtest.h" +#include "test/video_codec_settings.h" + +namespace webrtc { +namespace testing { + +using ::testing::_; +using ::testing::NiceMock; +using ::testing::Return; + +class MockEncoder : public VideoEncoder { + public: + // TODO(nisse): Valid overrides commented out, because the gmock + // methods don't use any override declarations, and we want to avoid + // warnings from -Winconsistent-missing-override. See + // http://crbug.com/428099. + MockEncoder() {} + virtual ~MockEncoder() {} + + MOCK_METHOD3(InitEncode, + int32_t(const VideoCodec* codec_settings, + int32_t number_of_cores, + size_t max_payload_size)); + + MOCK_METHOD1(RegisterEncodeCompleteCallback, int32_t(EncodedImageCallback*)); + + MOCK_METHOD0(Release, int32_t()); + + MOCK_METHOD3( + Encode, + int32_t(const VideoFrame& inputImage, + const CodecSpecificInfo* codecSpecificInfo, + const std::vector* frame_types) /* override */); + + MOCK_CONST_METHOD0(GetEncoderInfo, VideoEncoder::EncoderInfo(void)); +}; + +TEST(EncoderSimulcastProxy, ChoosesCorrectImplementation) { + const std::string kImplementationName = "Fake"; + const std::string kSimulcastAdaptedImplementationName = + "SimulcastEncoderAdapter (Fake, Fake, Fake)"; + VideoCodec codec_settings; + webrtc::test::CodecSettings(kVideoCodecVP8, &codec_settings); + codec_settings.simulcastStream[0] = {test::kTestWidth, + test::kTestHeight, + test::kTestFrameRate, + 2, + 2000, + 1000, + 1000, + 56}; + codec_settings.simulcastStream[1] = {test::kTestWidth, + test::kTestHeight, + test::kTestFrameRate, + 2, + 3000, + 1000, + 1000, + 56}; + codec_settings.simulcastStream[2] = {test::kTestWidth, + test::kTestHeight, + test::kTestFrameRate, + 2, + 5000, + 1000, + 1000, + 56}; + codec_settings.numberOfSimulcastStreams = 3; + + NiceMock* mock_encoder = new NiceMock(); + NiceMock simulcast_factory; + + EXPECT_CALL(*mock_encoder, InitEncode(_, _, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + VideoEncoder::EncoderInfo encoder_info; + encoder_info.implementation_name = kImplementationName; + EXPECT_CALL(*mock_encoder, GetEncoderInfo()) + .WillRepeatedly(Return(encoder_info)); + + EXPECT_CALL(simulcast_factory, CreateVideoEncoderProxy(_)) + .Times(1) + .WillOnce(Return(mock_encoder)); + + EncoderSimulcastProxy simulcast_enabled_proxy(&simulcast_factory, + SdpVideoFormat("VP8")); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + simulcast_enabled_proxy.InitEncode(&codec_settings, 4, 1200)); + EXPECT_EQ(kImplementationName, + simulcast_enabled_proxy.GetEncoderInfo().implementation_name); + + NiceMock* mock_encoder1 = new NiceMock(); + NiceMock* mock_encoder2 = new NiceMock(); + NiceMock* mock_encoder3 = new NiceMock(); + NiceMock* mock_encoder4 = new NiceMock(); + NiceMock nonsimulcast_factory; + + EXPECT_CALL(*mock_encoder1, InitEncode(_, _, _)) + .WillOnce( + Return(WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED)); + EXPECT_CALL(*mock_encoder1, GetEncoderInfo()) + .WillRepeatedly(Return(encoder_info)); + + EXPECT_CALL(*mock_encoder2, InitEncode(_, _, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_CALL(*mock_encoder2, GetEncoderInfo()) + .WillRepeatedly(Return(encoder_info)); + + EXPECT_CALL(*mock_encoder3, InitEncode(_, _, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_CALL(*mock_encoder3, GetEncoderInfo()) + .WillRepeatedly(Return(encoder_info)); + + EXPECT_CALL(*mock_encoder4, InitEncode(_, _, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + EXPECT_CALL(*mock_encoder4, GetEncoderInfo()) + .WillRepeatedly(Return(encoder_info)); + + EXPECT_CALL(nonsimulcast_factory, CreateVideoEncoderProxy(_)) + .Times(4) + .WillOnce(Return(mock_encoder1)) + .WillOnce(Return(mock_encoder2)) + .WillOnce(Return(mock_encoder3)) + .WillOnce(Return(mock_encoder4)); + + EncoderSimulcastProxy simulcast_disabled_proxy(&nonsimulcast_factory, + SdpVideoFormat("VP8")); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + simulcast_disabled_proxy.InitEncode(&codec_settings, 4, 1200)); + EXPECT_EQ(kSimulcastAdaptedImplementationName, + simulcast_disabled_proxy.GetEncoderInfo().implementation_name); + + // Cleanup. + simulcast_enabled_proxy.Release(); + simulcast_disabled_proxy.Release(); +} + +TEST(EncoderSimulcastProxy, ForwardsTrustedSetting) { + NiceMock* mock_encoder = new NiceMock(); + NiceMock simulcast_factory; + + EXPECT_CALL(*mock_encoder, InitEncode(_, _, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + + EXPECT_CALL(simulcast_factory, CreateVideoEncoderProxy(_)) + .Times(1) + .WillOnce(Return(mock_encoder)); + + EncoderSimulcastProxy simulcast_enabled_proxy(&simulcast_factory, + SdpVideoFormat("VP8")); + VideoCodec codec_settings; + webrtc::test::CodecSettings(kVideoCodecVP8, &codec_settings); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + simulcast_enabled_proxy.InitEncode(&codec_settings, 4, 1200)); + + VideoEncoder::EncoderInfo info; + info.has_trusted_rate_controller = true; + EXPECT_CALL(*mock_encoder, GetEncoderInfo()).WillRepeatedly(Return(info)); + + EXPECT_TRUE( + simulcast_enabled_proxy.GetEncoderInfo().has_trusted_rate_controller); +} + +TEST(EncoderSimulcastProxy, ForwardsHardwareAccelerated) { + NiceMock* mock_encoder = new NiceMock(); + NiceMock simulcast_factory; + + EXPECT_CALL(*mock_encoder, InitEncode(_, _, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + + EXPECT_CALL(simulcast_factory, CreateVideoEncoderProxy(_)) + .Times(1) + .WillOnce(Return(mock_encoder)); + + EncoderSimulcastProxy simulcast_enabled_proxy(&simulcast_factory, + SdpVideoFormat("VP8")); + VideoCodec codec_settings; + webrtc::test::CodecSettings(kVideoCodecVP8, &codec_settings); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + simulcast_enabled_proxy.InitEncode(&codec_settings, 4, 1200)); + + VideoEncoder::EncoderInfo info; + + info.is_hardware_accelerated = false; + EXPECT_CALL(*mock_encoder, GetEncoderInfo()).WillOnce(Return(info)); + EXPECT_FALSE( + simulcast_enabled_proxy.GetEncoderInfo().is_hardware_accelerated); + + info.is_hardware_accelerated = true; + EXPECT_CALL(*mock_encoder, GetEncoderInfo()).WillOnce(Return(info)); + EXPECT_TRUE(simulcast_enabled_proxy.GetEncoderInfo().is_hardware_accelerated); +} + +TEST(EncoderSimulcastProxy, ForwardsInternalSource) { + NiceMock* mock_encoder = new NiceMock(); + NiceMock simulcast_factory; + + EXPECT_CALL(*mock_encoder, InitEncode(_, _, _)) + .WillOnce(Return(WEBRTC_VIDEO_CODEC_OK)); + + EXPECT_CALL(simulcast_factory, CreateVideoEncoderProxy(_)) + .Times(1) + .WillOnce(Return(mock_encoder)); + + EncoderSimulcastProxy simulcast_enabled_proxy(&simulcast_factory, + SdpVideoFormat("VP8")); + VideoCodec codec_settings; + webrtc::test::CodecSettings(kVideoCodecVP8, &codec_settings); + EXPECT_EQ(WEBRTC_VIDEO_CODEC_OK, + simulcast_enabled_proxy.InitEncode(&codec_settings, 4, 1200)); + + VideoEncoder::EncoderInfo info; + + info.has_internal_source = false; + EXPECT_CALL(*mock_encoder, GetEncoderInfo()).WillOnce(Return(info)); + EXPECT_FALSE(simulcast_enabled_proxy.GetEncoderInfo().has_internal_source); + + info.has_internal_source = true; + EXPECT_CALL(*mock_encoder, GetEncoderInfo()).WillOnce(Return(info)); + EXPECT_TRUE(simulcast_enabled_proxy.GetEncoderInfo().has_internal_source); +} + +} // namespace testing +} // namespace webrtc diff --git a/media/engine/vp8_encoder_simulcast_proxy.h b/media/engine/vp8_encoder_simulcast_proxy.h index 25ef4a591a..c6973b82ac 100644 --- a/media/engine/vp8_encoder_simulcast_proxy.h +++ b/media/engine/vp8_encoder_simulcast_proxy.h @@ -30,6 +30,7 @@ namespace webrtc { // This class provides fallback to SimulcastEncoderAdapter if default VP8Encoder // doesn't support simulcast for provided settings. +// Deprecated: Use EncoderSimulcastProxy instead! class VP8EncoderSimulcastProxy : public VideoEncoder { public: VP8EncoderSimulcastProxy(VideoEncoderFactory* factory, diff --git a/video/BUILD.gn b/video/BUILD.gn index 4a3efa25c0..af08bb7827 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -244,8 +244,8 @@ if (rtc_include_tests) { "../logging:rtc_event_log_api", "../logging:rtc_event_log_impl_output", "../media:rtc_audio_video", + "../media:rtc_encoder_simulcast_proxy", "../media:rtc_internal_video_codecs", - "../media:rtc_vp8_encoder_simulcast_proxy", "../modules/audio_device:audio_device_api", "../modules/audio_device:audio_device_module_from_input_and_output", "../modules/audio_device:windows_core_audio_utility", @@ -269,6 +269,7 @@ if (rtc_include_tests) { "../test:test_support_test_artifacts", "../test:video_test_common", "../test:video_test_support", + "//third_party/abseil-cpp/absl/memory:memory", ] if (!build_with_chromium && is_clang) { # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). diff --git a/video/video_quality_test.cc b/video/video_quality_test.cc index d6ccb6522e..b5290d8463 100644 --- a/video/video_quality_test.cc +++ b/video/video_quality_test.cc @@ -16,13 +16,14 @@ #include #include +#include "absl/memory/memory.h" #include "api/video/builtin_video_bitrate_allocator_factory.h" #include "call/fake_network_pipe.h" #include "call/simulated_network.h" #include "logging/rtc_event_log/output/rtc_event_log_output_file.h" #include "media/engine/adm_helpers.h" +#include "media/engine/encoder_simulcast_proxy.h" #include "media/engine/internalencoderfactory.h" -#include "media/engine/vp8_encoder_simulcast_proxy.h" #include "media/engine/webrtcvideoengine.h" #include "modules/audio_device/include/audio_device.h" #include "modules/audio_mixer/audio_mixer_impl.h" @@ -187,7 +188,7 @@ std::unique_ptr VideoQualityTest::CreateVideoEncoder( VideoAnalyzer* analyzer) { std::unique_ptr encoder; if (format.name == "VP8") { - encoder = absl::make_unique( + encoder = absl::make_unique( &internal_encoder_factory_, format); } else if (format.name == "multiplex") { encoder = absl::make_unique(