diff --git a/api/BUILD.gn b/api/BUILD.gn index 41e1ad84e3..6ec5f23fd4 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -48,6 +48,7 @@ rtc_static_library("libjingle_peerconnection_api") { visibility = [ "*" ] cflags = [] sources = [ + "bitrate_constraints.h", "candidate.cc", "candidate.h", "cryptoparams.h", @@ -130,6 +131,48 @@ rtc_static_library("libjingle_peerconnection_api") { } } +rtc_source_set("video_quality_test_fixture_api") { + visibility = [ "*" ] + testonly = true + sources = [ + "test/video_quality_test_fixture.h", + ] + deps = [ + ":libjingle_peerconnection_api", + ":simulated_network_api", + "../call:fake_network", + "../call:rtp_interfaces", + "../test:test_common", + "../test:video_test_common", + "video_codecs:video_codecs_api", + ] + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } +} + +if (rtc_include_tests) { + rtc_source_set("create_video_quality_test_fixture_api") { + visibility = [ "*" ] + testonly = true + sources = [ + "test/create_video_quality_test_fixture.cc", + "test/create_video_quality_test_fixture.h", + ] + deps = [ + ":fec_controller_api", + ":video_quality_test_fixture_api", + "../rtc_base:ptr_util", + "../video:video_quality_test", + ] + if (!build_with_chromium && is_clang) { + # Suppress warnings from the Chromium Clang plugin (bugs.webrtc.org/163). + suppressed_configs += [ "//build/config/clang:find_bad_constructs" ] + } + } +} + rtc_source_set("libjingle_logging_api") { visibility = [ "*" ] sources = [ @@ -207,6 +250,18 @@ rtc_source_set("transport_api") { ] } +rtc_source_set("simulated_network_api") { + visibility = [ "*" ] + sources = [ + "test/simulated_network.h", + ] + deps = [ + ":optional", + "../rtc_base:criticalsection", + "../rtc_base:rtc_base", + ] +} + rtc_source_set("fec_controller_api") { visibility = [ "*" ] sources = [ diff --git a/call/bitrate_constraints.h b/api/bitrate_constraints.h similarity index 90% rename from call/bitrate_constraints.h rename to api/bitrate_constraints.h index 3da97a0b23..98e89c0858 100644 --- a/call/bitrate_constraints.h +++ b/api/bitrate_constraints.h @@ -8,8 +8,8 @@ * be found in the AUTHORS file in the root of the source tree. */ -#ifndef CALL_BITRATE_CONSTRAINTS_H_ -#define CALL_BITRATE_CONSTRAINTS_H_ +#ifndef API_BITRATE_CONSTRAINTS_H_ +#define API_BITRATE_CONSTRAINTS_H_ #include @@ -38,4 +38,4 @@ static T MinPositive(T a, T b) { return std::min(a, b); } } // namespace webrtc -#endif // CALL_BITRATE_CONSTRAINTS_H_ +#endif // API_BITRATE_CONSTRAINTS_H_ diff --git a/api/mediatypes.h b/api/mediatypes.h index 076fc0420c..f281276025 100644 --- a/api/mediatypes.h +++ b/api/mediatypes.h @@ -13,6 +13,9 @@ #include +// The cricket and webrtc have separate definitions for what a media type is. +// They're not compatible. Watch out for this. + namespace cricket { enum MediaType { MEDIA_TYPE_AUDIO, MEDIA_TYPE_VIDEO, MEDIA_TYPE_DATA }; @@ -24,4 +27,10 @@ MediaType MediaTypeFromString(const std::string& type_str); } // namespace cricket +namespace webrtc { + +enum class MediaType { ANY, AUDIO, VIDEO, DATA }; + +} // namespace webrtc + #endif // API_MEDIATYPES_H_ diff --git a/api/test/DEPS b/api/test/DEPS index 0a269fe959..98b1ad391f 100644 --- a/api/test/DEPS +++ b/api/test/DEPS @@ -2,4 +2,7 @@ specific_include_rules = { ".*": [ "+modules/video_coding", ], + ".*": [ + "+video" + ], } diff --git a/api/test/create_video_quality_test_fixture.cc b/api/test/create_video_quality_test_fixture.cc new file mode 100644 index 0000000000..bc7186152a --- /dev/null +++ b/api/test/create_video_quality_test_fixture.cc @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include +#include + +#include "api/test/create_video_quality_test_fixture.h" +#include "video/video_quality_test.h" +#include "rtc_base/ptr_util.h" + +namespace webrtc { + +std::unique_ptr +CreateVideoQualityTestFixture() { + // By default, we don't override the FEC module, so pass an empty factory. + return rtc::MakeUnique(nullptr); +} + +std::unique_ptr +CreateVideoQualityTestFixture( + std::unique_ptr fec_controller_factory) { + return rtc::MakeUnique(std::move(fec_controller_factory)); +} + +} // namespace webrtc + + diff --git a/api/test/create_video_quality_test_fixture.h b/api/test/create_video_quality_test_fixture.h new file mode 100644 index 0000000000..07c222f16a --- /dev/null +++ b/api/test/create_video_quality_test_fixture.h @@ -0,0 +1,29 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ +#ifndef API_TEST_CREATE_VIDEO_QUALITY_TEST_FIXTURE_H_ +#define API_TEST_CREATE_VIDEO_QUALITY_TEST_FIXTURE_H_ + +#include + +#include "api/fec_controller.h" +#include "api/test/video_quality_test_fixture.h" + +namespace webrtc { + +std::unique_ptr +CreateVideoQualityTestFixture(); + +std::unique_ptr +CreateVideoQualityTestFixture( + std::unique_ptr fec_controller_factory); + +} + +#endif // API_TEST_CREATE_VIDEO_QUALITY_TEST_FIXTURE_H_ diff --git a/api/test/simulated_network.h b/api/test/simulated_network.h new file mode 100644 index 0000000000..0006be2bc2 --- /dev/null +++ b/api/test/simulated_network.h @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_SIMULATED_NETWORK_H_ +#define API_TEST_SIMULATED_NETWORK_H_ + +#include +#include +#include +#include +#include + +#include "api/optional.h" +#include "rtc_base/criticalsection.h" +#include "rtc_base/random.h" +#include "rtc_base/thread_annotations.h" + +namespace webrtc { + +struct PacketInFlightInfo { + PacketInFlightInfo(size_t size, int64_t send_time_us, uint64_t packet_id) + : size(size), send_time_us(send_time_us), packet_id(packet_id) {} + + size_t size; + int64_t send_time_us; + // Unique identifier for the packet in relation to other packets in flight. + uint64_t packet_id; +}; + +struct PacketDeliveryInfo { + static constexpr int kNotReceived = -1; + PacketDeliveryInfo(PacketInFlightInfo source, int64_t receive_time_us) + : receive_time_us(receive_time_us), packet_id(source.packet_id) {} + int64_t receive_time_us; + uint64_t packet_id; +}; + +class NetworkSimulationInterface { + public: + // TODO(phoglund): this one shouldn't really be here; make fake network pipes + // injectable instead in the video quality test fixture. + struct SimulatedNetworkConfig { + SimulatedNetworkConfig() {} + // Queue length in number of packets. + size_t queue_length_packets = 0; + // Delay in addition to capacity induced delay. + int queue_delay_ms = 0; + // Standard deviation of the extra delay. + int delay_standard_deviation_ms = 0; + // Link capacity in kbps. + int link_capacity_kbps = 0; + // Random packet loss. + int loss_percent = 0; + // If packets are allowed to be reordered. + bool allow_reordering = false; + // The average length of a burst of lost packets. + int avg_burst_loss_length = -1; + }; + + virtual bool EnqueuePacket(PacketInFlightInfo packet_info) = 0; + // Retrieves all packets that should be delivered by the given receive time. + virtual std::vector DequeueDeliverablePackets( + int64_t receive_time_us) = 0; + virtual absl::optional NextDeliveryTimeUs() const = 0; + virtual ~NetworkSimulationInterface() = default; +}; + +} // namespace webrtc + +#endif // API_TEST_SIMULATED_NETWORK_H_ diff --git a/api/test/video_quality_test_fixture.h b/api/test/video_quality_test_fixture.h new file mode 100644 index 0000000000..b8a268cd3a --- /dev/null +++ b/api/test/video_quality_test_fixture.h @@ -0,0 +1,111 @@ +/* + * Copyright (c) 2018 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef API_TEST_VIDEO_QUALITY_TEST_FIXTURE_H_ +#define API_TEST_VIDEO_QUALITY_TEST_FIXTURE_H_ + +#include +#include +#include +#include + +#include "api/bitrate_constraints.h" +#include "api/mediatypes.h" +#include "api/test/simulated_network.h" +#include "api/video_codecs/video_encoder_config.h" + +namespace webrtc { + +class VideoQualityTestFixtureInterface { + public: + // Parameters are grouped into smaller structs to make it easier to set + // the desired elements and skip unused, using aggregate initialization. + // Unfortunately, C++11 (as opposed to C11) doesn't support unnamed structs, + // which makes the implementation of VideoQualityTest a bit uglier. + struct Params { + Params(); + ~Params(); + struct CallConfig { + bool send_side_bwe; + BitrateConstraints call_bitrate_config; + int num_thumbnails; + // Indicates if secondary_(video|ss|screenshare) structures are used. + bool dual_video; + } call; + struct Video { + bool enabled; + size_t width; + size_t height; + int32_t fps; + int min_bitrate_bps; + int target_bitrate_bps; + int max_bitrate_bps; + bool suspend_below_min_bitrate; + std::string codec; + int num_temporal_layers; + int selected_tl; + int min_transmit_bps; + bool ulpfec; + bool flexfec; + bool automatic_scaling; + std::string clip_name; // "Generator" to generate frames instead. + size_t capture_device_index; + } video[2]; + struct Audio { + bool enabled; + bool sync_video; + bool dtx; + } audio; + struct Screenshare { + bool enabled; + bool generate_slides; + int32_t slide_change_interval; + int32_t scroll_duration; + std::vector slides; + } screenshare[2]; + struct Analyzer { + std::string test_label; + double avg_psnr_threshold; // (*) + double avg_ssim_threshold; // (*) + int test_durations_secs; + std::string graph_data_output_filename; + std::string graph_title; + } analyzer; + NetworkSimulationInterface::SimulatedNetworkConfig pipe; + struct SS { // Spatial scalability. + std::vector streams; // If empty, one stream is assumed. + size_t selected_stream; + int num_spatial_layers; + int selected_sl; + InterLayerPredMode inter_layer_pred; + // If empty, bitrates are generated in VP9Impl automatically. + std::vector spatial_layers; + // If set, default parameters will be used instead of |streams|. + bool infer_streams; + } ss[2]; + struct Logging { + bool logs; + std::string rtc_event_log_name; + std::string rtp_dump_name; + std::string encoded_frame_base_path; + } logging; + }; + + virtual ~VideoQualityTestFixtureInterface() = default; + + virtual void RunWithAnalyzer(const Params& params) = 0; + virtual void RunWithRenderers(const Params& params) = 0; + + virtual const std::map& payload_type_map() = 0; +}; + +} // namespace webrtc + +#endif // API_TEST_VIDEO_QUALITY_TEST_FIXTURE_H_ diff --git a/call/BUILD.gn b/call/BUILD.gn index ff72fd96b0..dc55ab29b2 100644 --- a/call/BUILD.gn +++ b/call/BUILD.gn @@ -51,7 +51,6 @@ rtc_source_set("call_interfaces") { # when interfaces have stabilized. See also TODO for |mock_rtp_interfaces|. rtc_source_set("rtp_interfaces") { sources = [ - "bitrate_constraints.h", "rtcp_packet_sink_interface.h", "rtp_config.cc", "rtp_config.h", @@ -61,6 +60,7 @@ rtc_source_set("rtp_interfaces") { ] deps = [ "../api:array_view", + "../api:libjingle_peerconnection_api", "../api/transport:bitrate_settings", "../rtc_base:rtc_base_approved", "//third_party/abseil-cpp/absl/types:optional", @@ -123,6 +123,7 @@ rtc_source_set("bitrate_configurator") { ] deps = [ ":rtp_interfaces", + "../api:libjingle_peerconnection_api", "../api/transport:bitrate_settings", "../rtc_base:checks", "../rtc_base:rtc_base_approved", @@ -240,6 +241,7 @@ rtc_source_set("fake_network") { ":call_interfaces", "..:typedefs", "..:webrtc_common", + "../api:simulated_network_api", "../api:transport_api", "../modules:module_api", "../rtc_base:rtc_base_approved", @@ -366,6 +368,7 @@ if (rtc_include_tests) { ] deps = [ ":rtp_interfaces", + "../api:libjingle_peerconnection_api", "../modules/congestion_controller", "../modules/pacing", "../rtc_base:rate_limiter", diff --git a/call/call.h b/call/call.h index 5f700b614f..94af078182 100644 --- a/call/call.h +++ b/call/call.h @@ -15,6 +15,7 @@ #include #include +#include "api/mediatypes.h" #include "call/audio_receive_stream.h" #include "call/audio_send_stream.h" #include "call/call_config.h" @@ -30,8 +31,6 @@ namespace webrtc { -enum class MediaType { ANY, AUDIO, VIDEO, DATA }; - class PacketReceiver { public: enum DeliveryStatus { diff --git a/call/call_config.h b/call/call_config.h index 927ad75959..438929f4d3 100644 --- a/call/call_config.h +++ b/call/call_config.h @@ -10,11 +10,11 @@ #ifndef CALL_CALL_CONFIG_H_ #define CALL_CALL_CONFIG_H_ +#include "api/bitrate_constraints.h" #include "api/fec_controller.h" #include "api/rtcerror.h" #include "api/transport/network_control.h" #include "call/audio_state.h" -#include "call/bitrate_constraints.h" #include "rtc_base/platform_file.h" namespace webrtc { diff --git a/call/fake_network_pipe.h b/call/fake_network_pipe.h index c587973c71..866c5504d6 100644 --- a/call/fake_network_pipe.h +++ b/call/fake_network_pipe.h @@ -20,6 +20,7 @@ #include #include "api/call/transport.h" +#include "api/test/simulated_network.h" #include "call/call.h" #include "common_types.h" // NOLINT(build/include) #include "modules/include/module.h" @@ -84,56 +85,12 @@ class NetworkPacket { absl::optional packet_time_; }; -struct PacketInFlightInfo { - PacketInFlightInfo(size_t size, int64_t send_time_us, uint64_t packet_id) - : size(size), send_time_us(send_time_us), packet_id(packet_id) {} - - size_t size; - int64_t send_time_us; - // Unique identifier for the packet in relation to other packets in flight. - uint64_t packet_id; -}; - -struct PacketDeliveryInfo { - static constexpr int kNotReceived = -1; - PacketDeliveryInfo(PacketInFlightInfo source, int64_t receive_time_us) - : receive_time_us(receive_time_us), packet_id(source.packet_id) {} - int64_t receive_time_us; - uint64_t packet_id; -}; - -class NetworkSimulationInterface { - public: - virtual bool EnqueuePacket(PacketInFlightInfo packet_info) = 0; - // Retrieves all packets that should be delivered by the given receive time. - virtual std::vector DequeueDeliverablePackets( - int64_t receive_time_us) = 0; - virtual absl::optional NextDeliveryTimeUs() const = 0; - virtual ~NetworkSimulationInterface() = default; -}; - // Class simulating a network link. This is a simple and naive solution just // faking capacity and adding an extra transport delay in addition to the // capacity introduced delay. class SimulatedNetwork : public NetworkSimulationInterface { public: - struct Config { - Config() {} - // Queue length in number of packets. - size_t queue_length_packets = 0; - // Delay in addition to capacity induced delay. - int queue_delay_ms = 0; - // Standard deviation of the extra delay. - int delay_standard_deviation_ms = 0; - // Link capacity in kbps. - int link_capacity_kbps = 0; - // Random packet loss. - int loss_percent = 0; - // If packets are allowed to be reordered. - bool allow_reordering = false; - // The average length of a burst of lost packets. - int avg_burst_loss_length = -1; - }; + using Config = NetworkSimulationInterface::SimulatedNetworkConfig; explicit SimulatedNetwork(Config config, uint64_t random_seed = 1); // Sets a new configuration. This won't affect packets already in the pipe. diff --git a/call/rtp_bitrate_configurator.h b/call/rtp_bitrate_configurator.h index a311774ecf..07fe8d41a6 100644 --- a/call/rtp_bitrate_configurator.h +++ b/call/rtp_bitrate_configurator.h @@ -11,8 +11,8 @@ #ifndef CALL_RTP_BITRATE_CONFIGURATOR_H_ #define CALL_RTP_BITRATE_CONFIGURATOR_H_ +#include "api/bitrate_constraints.h" #include "api/transport/bitrate_settings.h" -#include "call/bitrate_constraints.h" #include "rtc_base/constructormagic.h" namespace webrtc { diff --git a/call/rtp_transport_controller_send_interface.h b/call/rtp_transport_controller_send_interface.h index 0269b13ad4..65b9d7d840 100644 --- a/call/rtp_transport_controller_send_interface.h +++ b/call/rtp_transport_controller_send_interface.h @@ -16,8 +16,8 @@ #include #include "absl/types/optional.h" +#include "api/bitrate_constraints.h" #include "api/transport/bitrate_settings.h" -#include "call/bitrate_constraints.h" namespace rtc { struct SentPacket; diff --git a/call/test/mock_rtp_transport_controller_send.h b/call/test/mock_rtp_transport_controller_send.h index e28b42ef26..e7c27d6f43 100644 --- a/call/test/mock_rtp_transport_controller_send.h +++ b/call/test/mock_rtp_transport_controller_send.h @@ -13,7 +13,7 @@ #include -#include "call/bitrate_constraints.h" +#include "api/bitrate_constraints.h" #include "call/rtp_transport_controller_send_interface.h" #include "modules/congestion_controller/include/network_changed_observer.h" #include "modules/pacing/packet_router.h" diff --git a/pc/webrtcsdp.cc b/pc/webrtcsdp.cc index d068470086..70a06343fd 100644 --- a/pc/webrtcsdp.cc +++ b/pc/webrtcsdp.cc @@ -22,6 +22,7 @@ #include #include +#include "api/mediatypes.h" #include "api/candidate.h" #include "api/cryptoparams.h" #include "api/jsepicecandidate.h" @@ -232,7 +233,7 @@ template static void AddFmtpLine(const T& codec, std::string* message); static void BuildMediaDescription(const ContentInfo* content_info, const TransportInfo* transport_info, - const MediaType media_type, + const cricket::MediaType media_type, const std::vector& candidates, int msid_signaling, std::string* message); @@ -240,11 +241,11 @@ static void BuildSctpContentAttributes(std::string* message, int sctp_port, bool use_sctpmap); static void BuildRtpContentAttributes(const MediaContentDescription* media_desc, - const MediaType media_type, + const cricket::MediaType media_type, int msid_signaling, std::string* message); static void BuildRtpMap(const MediaContentDescription* media_desc, - const MediaType media_type, + const cricket::MediaType media_type, std::string* message); static void BuildCandidate(const std::vector& candidates, bool include_ufrag, @@ -275,7 +276,7 @@ static bool ParseMediaDescription( std::vector* candidates, SdpParseError* error); static bool ParseContent(const std::string& message, - const MediaType media_type, + const cricket::MediaType media_type, int mline_index, const std::string& protocol, const std::vector& payload_types, @@ -298,12 +299,12 @@ static bool ParseCryptoAttribute(const std::string& line, MediaContentDescription* media_desc, SdpParseError* error); static bool ParseRtpmapAttribute(const std::string& line, - const MediaType media_type, + const cricket::MediaType media_type, const std::vector& payload_types, MediaContentDescription* media_desc, SdpParseError* error); static bool ParseFmtpAttributes(const std::string& line, - const MediaType media_type, + const cricket::MediaType media_type, MediaContentDescription* media_desc, SdpParseError* error); static bool ParseFmtpParam(const std::string& line, @@ -315,7 +316,7 @@ static bool ParseCandidate(const std::string& message, SdpParseError* error, bool is_raw); static bool ParseRtcpFbAttribute(const std::string& line, - const MediaType media_type, + const cricket::MediaType media_type, MediaContentDescription* media_desc, SdpParseError* error); static bool ParseIceOptions(const std::string& line, @@ -1243,7 +1244,7 @@ bool ParseExtmap(const std::string& line, void BuildMediaDescription(const ContentInfo* content_info, const TransportInfo* transport_info, - const MediaType media_type, + const cricket::MediaType media_type, const std::vector& candidates, int msid_signaling, std::string* message) { @@ -1477,7 +1478,7 @@ void BuildSctpContentAttributes(std::string* message, } void BuildRtpContentAttributes(const MediaContentDescription* media_desc, - const MediaType media_type, + const cricket::MediaType media_type, int msid_signaling, std::string* message) { std::ostringstream os; @@ -1767,7 +1768,7 @@ bool GetParameter(const std::string& name, } void BuildRtpMap(const MediaContentDescription* media_desc, - const MediaType media_type, + const cricket::MediaType media_type, std::string* message) { RTC_DCHECK(message != NULL); RTC_DCHECK(media_desc != NULL); @@ -2328,7 +2329,7 @@ void MaybeCreateStaticPayloadAudioCodecs(const std::vector& fmts, template static C* ParseContentDescription(const std::string& message, - const MediaType media_type, + const cricket::MediaType media_type, int mline_index, const std::string& protocol, const std::vector& payload_types, @@ -2709,7 +2710,7 @@ void AddAudioAttribute(const std::string& name, } bool ParseContent(const std::string& message, - const MediaType media_type, + const cricket::MediaType media_type, int mline_index, const std::string& protocol, const std::vector& payload_types, @@ -3198,7 +3199,7 @@ void UpdateCodec(int payload_type, } bool ParseRtpmapAttribute(const std::string& line, - const MediaType media_type, + const cricket::MediaType media_type, const std::vector& payload_types, MediaContentDescription* media_desc, SdpParseError* error) { @@ -3281,7 +3282,7 @@ bool ParseFmtpParam(const std::string& line, } bool ParseFmtpAttributes(const std::string& line, - const MediaType media_type, + const cricket::MediaType media_type, MediaContentDescription* media_desc, SdpParseError* error) { if (media_type != cricket::MEDIA_TYPE_AUDIO && @@ -3345,7 +3346,7 @@ bool ParseFmtpAttributes(const std::string& line, } bool ParseRtcpFbAttribute(const std::string& line, - const MediaType media_type, + const cricket::MediaType media_type, MediaContentDescription* media_desc, SdpParseError* error) { if (media_type != cricket::MEDIA_TYPE_AUDIO && diff --git a/video/BUILD.gn b/video/BUILD.gn index df50d90687..113caeae24 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -146,12 +146,20 @@ if (rtc_include_tests) { } rtc_source_set("video_quality_test") { testonly = true - visibility = [ ":*" ] # Only targets in this file can depend on this. + + # Only targets in this file and api/ can depend on this. + visibility = [ + ":*", + "../api:create_video_quality_test_fixture_api", + ] sources = [ "video_quality_test.cc", "video_quality_test.h", ] deps = [ + "../api:fec_controller_api", + "../api:video_quality_test_fixture_api", + "../call:fake_network", "../logging:rtc_event_log_api", "../logging:rtc_event_log_impl_output", "../media:rtc_audio_video", diff --git a/video/full_stack_tests.cc b/video/full_stack_tests.cc index 5dc209abcf..79bef26ef2 100644 --- a/video/full_stack_tests.cc +++ b/video/full_stack_tests.cc @@ -45,28 +45,35 @@ namespace webrtc { namespace { static const int kFullStackTestDurationSecs = 45; -} // namespace +const char kScreenshareSimulcastExperiment[] = + "WebRTC-SimulcastScreenshare/Enabled/"; +const char kRoundRobinPacingQueueExperiment[] = + "WebRTC-RoundRobinPacing/Enabled/"; +const char kPacerPushBackExperiment[] = + "WebRTC-PacerPushbackExperiment/Enabled/"; -class FullStackTest : public VideoQualityTest { +std::string AlrProbingExperimentName() { + auto experiment = std::string( + AlrExperimentSettings::kScreenshareProbingBweExperimentName); + return experiment + "/1.1,2875,85,20,-20,0/"; +} + +struct ParamsWithLogging : public VideoQualityTest::Params { public: - void RunTest(VideoQualityTest::Params params) { - params.logging = {flags::FLAG_logs, flags::RtcEventLogName(), - flags::RtpDumpName(), flags::EncodedFramePath()}; - RunWithAnalyzer(params); + ParamsWithLogging() { + // Use these logging flags by default, for everything. + logging = {flags::FLAG_logs, flags::RtcEventLogName(), + flags::RtpDumpName(), flags::EncodedFramePath()}; } - - protected: - const std::string kScreenshareSimulcastExperiment = - "WebRTC-SimulcastScreenshare/Enabled/"; - const std::string kAlrProbingExperiment = - std::string(AlrExperimentSettings::kScreenshareProbingBweExperimentName) + - "/1.1,2875,85,20,-20,0/"; - const std::string kRoundRobinPacingQueueExperiment = - "WebRTC-RoundRobinPacing/Enabled/"; - const std::string kPacerPushBackExperiment = - "WebRTC-PacerPushbackExperiment/Enabled/"; }; +std::unique_ptr +CreateVideoQualityTestFixture() { + return rtc::MakeUnique(nullptr); +} + +} // namespace + // VideoQualityTest::Params params = { // { ... }, // Common. // { ... }, // Video-specific settings. @@ -78,20 +85,21 @@ class FullStackTest : public VideoQualityTest { // }; #if !defined(RTC_DISABLE_VP9) -TEST_F(FullStackTest, ForemanCifWithoutPacketLossVp9) { - // TODO(pbos): Decide on psnr/ssim thresholds for foreman_cif. - VideoQualityTest::Params foreman_cif; +TEST(FullStackTest, ForemanCifWithoutPacketLossVp9) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging foreman_cif; foreman_cif.call.send_side_bwe = true; foreman_cif.video[0] = {true, 352, 288, 30, 700000, 700000, 700000, false, "VP9", 1, 0, 0, false, false, false, "foreman_cif"}; foreman_cif.analyzer = {"foreman_cif_net_delay_0_0_plr_0_VP9", 0.0, 0.0, kFullStackTestDurationSecs}; - RunTest(foreman_cif); + fixture->RunWithAnalyzer(foreman_cif); } -TEST_F(FullStackTest, ForemanCifPlr5Vp9) { - VideoQualityTest::Params foreman_cif; +TEST(FullStackTest, ForemanCifPlr5Vp9) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging foreman_cif; foreman_cif.call.send_side_bwe = true; foreman_cif.video[0] = {true, 352, 288, 30, 30000, 500000, 2000000, false, "VP9", 1, @@ -100,11 +108,12 @@ TEST_F(FullStackTest, ForemanCifPlr5Vp9) { kFullStackTestDurationSecs}; foreman_cif.pipe.loss_percent = 5; foreman_cif.pipe.queue_delay_ms = 50; - RunTest(foreman_cif); + fixture->RunWithAnalyzer(foreman_cif); } -TEST_F(FullStackTest, ForemanCifWithoutPacketLossMultiplexI420Frame) { - VideoQualityTest::Params foreman_cif; +TEST(FullStackTest, ForemanCifWithoutPacketLossMultiplexI420Frame) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging foreman_cif; foreman_cif.call.send_side_bwe = true; foreman_cif.video[0] = {true, 352, 288, 30, 700000, 700000, 700000, false, "multiplex", 1, @@ -112,11 +121,13 @@ TEST_F(FullStackTest, ForemanCifWithoutPacketLossMultiplexI420Frame) { "foreman_cif"}; foreman_cif.analyzer = {"foreman_cif_net_delay_0_0_plr_0_Multiplex", 0.0, 0.0, kFullStackTestDurationSecs}; - RunTest(foreman_cif); + fixture->RunWithAnalyzer(foreman_cif); } -TEST_F(FullStackTest, GeneratorWithoutPacketLossMultiplexI420AFrame) { - VideoQualityTest::Params generator; +TEST(FullStackTest, GeneratorWithoutPacketLossMultiplexI420AFrame) { + auto fixture = CreateVideoQualityTestFixture(); + + ParamsWithLogging generator; generator.call.send_side_bwe = true; generator.video[0] = {true, 352, 288, 30, 700000, 700000, 700000, false, "multiplex", 1, @@ -124,7 +135,7 @@ TEST_F(FullStackTest, GeneratorWithoutPacketLossMultiplexI420AFrame) { "GeneratorI420A"}; generator.analyzer = {"generator_net_delay_0_0_plr_0_Multiplex", 0.0, 0.0, kFullStackTestDurationSecs}; - RunTest(generator); + fixture->RunWithAnalyzer(generator); } #endif // !defined(RTC_DISABLE_VP9) @@ -135,42 +146,46 @@ TEST_F(FullStackTest, GeneratorWithoutPacketLossMultiplexI420AFrame) { #else #define MAYBE_ParisQcifWithoutPacketLoss ParisQcifWithoutPacketLoss #endif -TEST_F(FullStackTest, MAYBE_ParisQcifWithoutPacketLoss) { - VideoQualityTest::Params paris_qcif; +TEST(FullStackTest, MAYBE_ParisQcifWithoutPacketLoss) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging paris_qcif; paris_qcif.call.send_side_bwe = true; paris_qcif.video[0] = {true, 176, 144, 30, 300000, 300000, 300000, false, "VP8", 1, 0, 0, false, false, false, "paris_qcif"}; paris_qcif.analyzer = {"net_delay_0_0_plr_0", 36.0, 0.96, kFullStackTestDurationSecs}; - RunTest(paris_qcif); + fixture->RunWithAnalyzer(paris_qcif); } -TEST_F(FullStackTest, ForemanCifWithoutPacketLoss) { +TEST(FullStackTest, ForemanCifWithoutPacketLoss) { + auto fixture = CreateVideoQualityTestFixture(); // TODO(pbos): Decide on psnr/ssim thresholds for foreman_cif. - VideoQualityTest::Params foreman_cif; + ParamsWithLogging foreman_cif; foreman_cif.call.send_side_bwe = true; foreman_cif.video[0] = {true, 352, 288, 30, 700000, 700000, 700000, false, "VP8", 1, 0, 0, false, false, false, "foreman_cif"}; foreman_cif.analyzer = {"foreman_cif_net_delay_0_0_plr_0", 0.0, 0.0, kFullStackTestDurationSecs}; - RunTest(foreman_cif); + fixture->RunWithAnalyzer(foreman_cif); } -TEST_F(FullStackTest, ForemanCif30kbpsWithoutPacketLoss) { - VideoQualityTest::Params foreman_cif; +TEST(FullStackTest, ForemanCif30kbpsWithoutPacketLoss) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging foreman_cif; foreman_cif.call.send_side_bwe = true; foreman_cif.video[0] = {true, 352, 288, 10, 30000, 30000, 30000, false, "VP8", 1, 0, 0, false, false, false, "foreman_cif"}; foreman_cif.analyzer = {"foreman_cif_30kbps_net_delay_0_0_plr_0", 0.0, 0.0, kFullStackTestDurationSecs}; - RunTest(foreman_cif); + fixture->RunWithAnalyzer(foreman_cif); } // Link capacity below default start rate. Automatic down scaling enabled. -TEST_F(FullStackTest, ForemanCifLink150kbpsWithoutPacketLoss) { +TEST(FullStackTest, ForemanCifLink150kbpsWithoutPacketLoss) { + auto fixture = CreateVideoQualityTestFixture(); VideoQualityTest::Params foreman_cif; foreman_cif.call.send_side_bwe = true; foreman_cif.video[0] = {true, 352, 288, 30, 30000, @@ -180,11 +195,12 @@ TEST_F(FullStackTest, ForemanCifLink150kbpsWithoutPacketLoss) { 0.0, 0.0, kFullStackTestDurationSecs}; foreman_cif.pipe.link_capacity_kbps = 150; - RunTest(foreman_cif); + fixture->RunWithAnalyzer(foreman_cif); } -TEST_F(FullStackTest, ForemanCifPlr5) { - VideoQualityTest::Params foreman_cif; +TEST(FullStackTest, ForemanCifPlr5) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging foreman_cif; foreman_cif.call.send_side_bwe = true; foreman_cif.video[0] = {true, 352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1, @@ -193,11 +209,12 @@ TEST_F(FullStackTest, ForemanCifPlr5) { kFullStackTestDurationSecs}; foreman_cif.pipe.loss_percent = 5; foreman_cif.pipe.queue_delay_ms = 50; - RunTest(foreman_cif); + fixture->RunWithAnalyzer(foreman_cif); } -TEST_F(FullStackTest, ForemanCifPlr5Ulpfec) { - VideoQualityTest::Params foreman_cif; +TEST(FullStackTest, ForemanCifPlr5Ulpfec) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging foreman_cif; foreman_cif.call.send_side_bwe = true; foreman_cif.video[0] = {true, 352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1, @@ -206,11 +223,12 @@ TEST_F(FullStackTest, ForemanCifPlr5Ulpfec) { kFullStackTestDurationSecs}; foreman_cif.pipe.loss_percent = 5; foreman_cif.pipe.queue_delay_ms = 50; - RunTest(foreman_cif); + fixture->RunWithAnalyzer(foreman_cif); } -TEST_F(FullStackTest, ForemanCifPlr5Flexfec) { - VideoQualityTest::Params foreman_cif; +TEST(FullStackTest, ForemanCifPlr5Flexfec) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging foreman_cif; foreman_cif.call.send_side_bwe = true; foreman_cif.video[0] = {true, 352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1, @@ -219,11 +237,12 @@ TEST_F(FullStackTest, ForemanCifPlr5Flexfec) { kFullStackTestDurationSecs}; foreman_cif.pipe.loss_percent = 5; foreman_cif.pipe.queue_delay_ms = 50; - RunTest(foreman_cif); + fixture->RunWithAnalyzer(foreman_cif); } -TEST_F(FullStackTest, ForemanCif500kbpsPlr3Flexfec) { - VideoQualityTest::Params foreman_cif; +TEST(FullStackTest, ForemanCif500kbpsPlr3Flexfec) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging foreman_cif; foreman_cif.call.send_side_bwe = true; foreman_cif.video[0] = {true, 352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1, @@ -233,11 +252,12 @@ TEST_F(FullStackTest, ForemanCif500kbpsPlr3Flexfec) { foreman_cif.pipe.loss_percent = 3; foreman_cif.pipe.link_capacity_kbps = 500; foreman_cif.pipe.queue_delay_ms = 50; - RunTest(foreman_cif); + fixture->RunWithAnalyzer(foreman_cif); } -TEST_F(FullStackTest, ForemanCif500kbpsPlr3Ulpfec) { - VideoQualityTest::Params foreman_cif; +TEST(FullStackTest, ForemanCif500kbpsPlr3Ulpfec) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging foreman_cif; foreman_cif.call.send_side_bwe = true; foreman_cif.video[0] = {true, 352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1, @@ -247,35 +267,38 @@ TEST_F(FullStackTest, ForemanCif500kbpsPlr3Ulpfec) { foreman_cif.pipe.loss_percent = 3; foreman_cif.pipe.link_capacity_kbps = 500; foreman_cif.pipe.queue_delay_ms = 50; - RunTest(foreman_cif); + fixture->RunWithAnalyzer(foreman_cif); } #if defined(WEBRTC_USE_H264) -TEST_F(FullStackTest, ForemanCifWithoutPacketlossH264) { +TEST(FullStackTest, ForemanCifWithoutPacketlossH264) { + auto fixture = CreateVideoQualityTestFixture(); // TODO(pbos): Decide on psnr/ssim thresholds for foreman_cif. - VideoQualityTest::Params foreman_cif; + ParamsWithLogging foreman_cif; foreman_cif.call.send_side_bwe = true; foreman_cif.video[0] = {true, 352, 288, 30, 700000, 700000, 700000, false, "H264", 1, 0, 0, false, false, false, "foreman_cif"}; foreman_cif.analyzer = {"foreman_cif_net_delay_0_0_plr_0_H264", 0.0, 0.0, kFullStackTestDurationSecs}; - RunTest(foreman_cif); + fixture->RunWithAnalyzer(foreman_cif); } -TEST_F(FullStackTest, ForemanCif30kbpsWithoutPacketlossH264) { - VideoQualityTest::Params foreman_cif; +TEST(FullStackTest, ForemanCif30kbpsWithoutPacketlossH264) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging foreman_cif; foreman_cif.call.send_side_bwe = true; foreman_cif.video[0] = {true, 352, 288, 10, 30000, 30000, 30000, false, "H264", 1, 0, 0, false, false, false, "foreman_cif"}; foreman_cif.analyzer = {"foreman_cif_30kbps_net_delay_0_0_plr_0_H264", 0.0, 0.0, kFullStackTestDurationSecs}; - RunTest(foreman_cif); + fixture->RunWithAnalyzer(foreman_cif); } -TEST_F(FullStackTest, ForemanCifPlr5H264) { - VideoQualityTest::Params foreman_cif; +TEST(FullStackTest, ForemanCifPlr5H264) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging foreman_cif; foreman_cif.call.send_side_bwe = true; foreman_cif.video[0] = {true, 352, 288, 30, 30000, 500000, 2000000, false, "H264", 1, @@ -284,14 +307,15 @@ TEST_F(FullStackTest, ForemanCifPlr5H264) { kFullStackTestDurationSecs}; foreman_cif.pipe.loss_percent = 5; foreman_cif.pipe.queue_delay_ms = 50; - RunTest(foreman_cif); + fixture->RunWithAnalyzer(foreman_cif); } -TEST_F(FullStackTest, ForemanCifPlr5H264SpsPpsIdrIsKeyframe) { +TEST(FullStackTest, ForemanCifPlr5H264SpsPpsIdrIsKeyframe) { + auto fixture = CreateVideoQualityTestFixture(); test::ScopedFieldTrials override_field_trials( "WebRTC-SpsPpsIdrIsH264Keyframe/Enabled/"); - VideoQualityTest::Params foreman_cif; + ParamsWithLogging foreman_cif; foreman_cif.call.send_side_bwe = true; foreman_cif.video[0] = {true, 352, 288, 30, 30000, 500000, 2000000, false, "H264", 1, @@ -300,12 +324,13 @@ TEST_F(FullStackTest, ForemanCifPlr5H264SpsPpsIdrIsKeyframe) { 0.0, kFullStackTestDurationSecs}; foreman_cif.pipe.loss_percent = 5; foreman_cif.pipe.queue_delay_ms = 50; - RunTest(foreman_cif); + fixture->RunWithAnalyzer(foreman_cif); } // Verify that this is worth the bot time, before enabling. -TEST_F(FullStackTest, ForemanCifPlr5H264Flexfec) { - VideoQualityTest::Params foreman_cif; +TEST(FullStackTest, ForemanCifPlr5H264Flexfec) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging foreman_cif; foreman_cif.call.send_side_bwe = true; foreman_cif.video[0] = {true, 352, 288, 30, 30000, 500000, 2000000, false, "H264", 1, @@ -314,13 +339,14 @@ TEST_F(FullStackTest, ForemanCifPlr5H264Flexfec) { kFullStackTestDurationSecs}; foreman_cif.pipe.loss_percent = 5; foreman_cif.pipe.queue_delay_ms = 50; - RunTest(foreman_cif); + fixture->RunWithAnalyzer(foreman_cif); } // Ulpfec with H264 is an unsupported combination, so this test is only useful // for debugging. It is therefore disabled by default. -TEST_F(FullStackTest, DISABLED_ForemanCifPlr5H264Ulpfec) { - VideoQualityTest::Params foreman_cif; +TEST(FullStackTest, DISABLED_ForemanCifPlr5H264Ulpfec) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging foreman_cif; foreman_cif.call.send_side_bwe = true; foreman_cif.video[0] = {true, 352, 288, 30, 30000, 500000, 2000000, false, "H264", 1, @@ -329,12 +355,13 @@ TEST_F(FullStackTest, DISABLED_ForemanCifPlr5H264Ulpfec) { kFullStackTestDurationSecs}; foreman_cif.pipe.loss_percent = 5; foreman_cif.pipe.queue_delay_ms = 50; - RunTest(foreman_cif); + fixture->RunWithAnalyzer(foreman_cif); } #endif // defined(WEBRTC_USE_H264) -TEST_F(FullStackTest, ForemanCif500kbps) { - VideoQualityTest::Params foreman_cif; +TEST(FullStackTest, ForemanCif500kbps) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging foreman_cif; foreman_cif.call.send_side_bwe = true; foreman_cif.video[0] = {true, 352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1, @@ -344,11 +371,12 @@ TEST_F(FullStackTest, ForemanCif500kbps) { foreman_cif.pipe.queue_length_packets = 0; foreman_cif.pipe.queue_delay_ms = 0; foreman_cif.pipe.link_capacity_kbps = 500; - RunTest(foreman_cif); + fixture->RunWithAnalyzer(foreman_cif); } -TEST_F(FullStackTest, ForemanCif500kbpsLimitedQueue) { - VideoQualityTest::Params foreman_cif; +TEST(FullStackTest, ForemanCif500kbpsLimitedQueue) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging foreman_cif; foreman_cif.call.send_side_bwe = true; foreman_cif.video[0] = {true, 352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1, @@ -358,11 +386,12 @@ TEST_F(FullStackTest, ForemanCif500kbpsLimitedQueue) { foreman_cif.pipe.queue_length_packets = 32; foreman_cif.pipe.queue_delay_ms = 0; foreman_cif.pipe.link_capacity_kbps = 500; - RunTest(foreman_cif); + fixture->RunWithAnalyzer(foreman_cif); } -TEST_F(FullStackTest, ForemanCif500kbps100ms) { - VideoQualityTest::Params foreman_cif; +TEST(FullStackTest, ForemanCif500kbps100ms) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging foreman_cif; foreman_cif.call.send_side_bwe = true; foreman_cif.video[0] = {true, 352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1, @@ -372,11 +401,12 @@ TEST_F(FullStackTest, ForemanCif500kbps100ms) { foreman_cif.pipe.queue_length_packets = 0; foreman_cif.pipe.queue_delay_ms = 100; foreman_cif.pipe.link_capacity_kbps = 500; - RunTest(foreman_cif); + fixture->RunWithAnalyzer(foreman_cif); } -TEST_F(FullStackTest, ForemanCif500kbps100msLimitedQueue) { - VideoQualityTest::Params foreman_cif; +TEST(FullStackTest, ForemanCif500kbps100msLimitedQueue) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging foreman_cif; foreman_cif.call.send_side_bwe = true; foreman_cif.video[0] = {true, 352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1, @@ -386,11 +416,12 @@ TEST_F(FullStackTest, ForemanCif500kbps100msLimitedQueue) { foreman_cif.pipe.queue_length_packets = 32; foreman_cif.pipe.queue_delay_ms = 100; foreman_cif.pipe.link_capacity_kbps = 500; - RunTest(foreman_cif); + fixture->RunWithAnalyzer(foreman_cif); } -TEST_F(FullStackTest, ForemanCif500kbps100msLimitedQueueRecvBwe) { - VideoQualityTest::Params foreman_cif; +TEST(FullStackTest, ForemanCif500kbps100msLimitedQueueRecvBwe) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging foreman_cif; foreman_cif.call.send_side_bwe = false; foreman_cif.video[0] = {true, 352, 288, 30, 30000, 500000, 2000000, false, "VP8", 1, @@ -400,11 +431,12 @@ TEST_F(FullStackTest, ForemanCif500kbps100msLimitedQueueRecvBwe) { foreman_cif.pipe.queue_length_packets = 32; foreman_cif.pipe.queue_delay_ms = 100; foreman_cif.pipe.link_capacity_kbps = 500; - RunTest(foreman_cif); + fixture->RunWithAnalyzer(foreman_cif); } -TEST_F(FullStackTest, ForemanCif1000kbps100msLimitedQueue) { - VideoQualityTest::Params foreman_cif; +TEST(FullStackTest, ForemanCif1000kbps100msLimitedQueue) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging foreman_cif; foreman_cif.call.send_side_bwe = true; foreman_cif.video[0] = {true, 352, 288, 30, 30000, 2000000, 2000000, false, "VP8", 1, @@ -414,12 +446,13 @@ TEST_F(FullStackTest, ForemanCif1000kbps100msLimitedQueue) { foreman_cif.pipe.queue_length_packets = 32; foreman_cif.pipe.queue_delay_ms = 100; foreman_cif.pipe.link_capacity_kbps = 1000; - RunTest(foreman_cif); + fixture->RunWithAnalyzer(foreman_cif); } // TODO(sprang): Remove this if we have the similar ModerateLimits below? -TEST_F(FullStackTest, ConferenceMotionHd2000kbps100msLimitedQueue) { - VideoQualityTest::Params conf_motion_hd; +TEST(FullStackTest, ConferenceMotionHd2000kbps100msLimitedQueue) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging conf_motion_hd; conf_motion_hd.call.send_side_bwe = true; conf_motion_hd.video[0] = { true, 1280, 720, 50, 30000, @@ -430,11 +463,12 @@ TEST_F(FullStackTest, ConferenceMotionHd2000kbps100msLimitedQueue) { conf_motion_hd.pipe.queue_length_packets = 32; conf_motion_hd.pipe.queue_delay_ms = 100; conf_motion_hd.pipe.link_capacity_kbps = 2000; - RunTest(conf_motion_hd); + fixture->RunWithAnalyzer(conf_motion_hd); } -TEST_F(FullStackTest, ConferenceMotionHd1TLModerateLimits) { - VideoQualityTest::Params conf_motion_hd; +TEST(FullStackTest, ConferenceMotionHd1TLModerateLimits) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging conf_motion_hd; conf_motion_hd.call.send_side_bwe = true; conf_motion_hd.video[0] = { true, 1280, 720, 50, 30000, @@ -446,11 +480,12 @@ TEST_F(FullStackTest, ConferenceMotionHd1TLModerateLimits) { conf_motion_hd.pipe.loss_percent = 3; conf_motion_hd.pipe.queue_delay_ms = 100; conf_motion_hd.pipe.link_capacity_kbps = 2000; - RunTest(conf_motion_hd); + fixture->RunWithAnalyzer(conf_motion_hd); } -TEST_F(FullStackTest, ConferenceMotionHd2TLModerateLimits) { - VideoQualityTest::Params conf_motion_hd; +TEST(FullStackTest, ConferenceMotionHd2TLModerateLimits) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging conf_motion_hd; conf_motion_hd.call.send_side_bwe = true; conf_motion_hd.video[0] = { true, 1280, 720, 50, 30000, @@ -462,11 +497,12 @@ TEST_F(FullStackTest, ConferenceMotionHd2TLModerateLimits) { conf_motion_hd.pipe.loss_percent = 3; conf_motion_hd.pipe.queue_delay_ms = 100; conf_motion_hd.pipe.link_capacity_kbps = 2000; - RunTest(conf_motion_hd); + fixture->RunWithAnalyzer(conf_motion_hd); } -TEST_F(FullStackTest, ConferenceMotionHd3TLModerateLimits) { - VideoQualityTest::Params conf_motion_hd; +TEST(FullStackTest, ConferenceMotionHd3TLModerateLimits) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging conf_motion_hd; conf_motion_hd.call.send_side_bwe = true; conf_motion_hd.video[0] = { true, 1280, 720, 50, 30000, @@ -478,11 +514,12 @@ TEST_F(FullStackTest, ConferenceMotionHd3TLModerateLimits) { conf_motion_hd.pipe.loss_percent = 3; conf_motion_hd.pipe.queue_delay_ms = 100; conf_motion_hd.pipe.link_capacity_kbps = 2000; - RunTest(conf_motion_hd); + fixture->RunWithAnalyzer(conf_motion_hd); } -TEST_F(FullStackTest, ConferenceMotionHd4TLModerateLimits) { - VideoQualityTest::Params conf_motion_hd; +TEST(FullStackTest, ConferenceMotionHd4TLModerateLimits) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging conf_motion_hd; conf_motion_hd.call.send_side_bwe = true; conf_motion_hd.video[0] = { true, 1280, 720, 50, 30000, @@ -494,12 +531,13 @@ TEST_F(FullStackTest, ConferenceMotionHd4TLModerateLimits) { conf_motion_hd.pipe.loss_percent = 3; conf_motion_hd.pipe.queue_delay_ms = 100; conf_motion_hd.pipe.link_capacity_kbps = 2000; - RunTest(conf_motion_hd); + fixture->RunWithAnalyzer(conf_motion_hd); } -TEST_F(FullStackTest, ConferenceMotionHd3TLModerateLimitsAltTLPattern) { +TEST(FullStackTest, ConferenceMotionHd3TLModerateLimitsAltTLPattern) { + auto fixture = CreateVideoQualityTestFixture(); test::ScopedFieldTrials field_trial("WebRTC-UseShortVP8TL3Pattern/Enabled/"); - VideoQualityTest::Params conf_motion_hd; + ParamsWithLogging conf_motion_hd; conf_motion_hd.call.send_side_bwe = true; conf_motion_hd.video[0] = { true, 1280, 720, 50, 30000, @@ -511,12 +549,13 @@ TEST_F(FullStackTest, ConferenceMotionHd3TLModerateLimitsAltTLPattern) { conf_motion_hd.pipe.loss_percent = 3; conf_motion_hd.pipe.queue_delay_ms = 100; conf_motion_hd.pipe.link_capacity_kbps = 2000; - RunTest(conf_motion_hd); + fixture->RunWithAnalyzer(conf_motion_hd); } #if !defined(RTC_DISABLE_VP9) -TEST_F(FullStackTest, ConferenceMotionHd2000kbps100msLimitedQueueVP9) { - VideoQualityTest::Params conf_motion_hd; +TEST(FullStackTest, ConferenceMotionHd2000kbps100msLimitedQueueVP9) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging conf_motion_hd; conf_motion_hd.call.send_side_bwe = true; conf_motion_hd.video[0] = { true, 1280, 720, 50, 30000, @@ -528,12 +567,13 @@ TEST_F(FullStackTest, ConferenceMotionHd2000kbps100msLimitedQueueVP9) { conf_motion_hd.pipe.queue_length_packets = 32; conf_motion_hd.pipe.queue_delay_ms = 100; conf_motion_hd.pipe.link_capacity_kbps = 2000; - RunTest(conf_motion_hd); + fixture->RunWithAnalyzer(conf_motion_hd); } #endif -TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL) { - VideoQualityTest::Params screenshare; +TEST(FullStackTest, ScreenshareSlidesVP8_2TL) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging screenshare; screenshare.call.send_side_bwe = true; screenshare.video[0] = {true, 1850, 1110, 5, 50000, 200000, 2000000, false, "VP8", 2, @@ -541,12 +581,13 @@ TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL) { screenshare.screenshare[0] = {true, false, 10}; screenshare.analyzer = {"screenshare_slides", 0.0, 0.0, kFullStackTestDurationSecs}; - RunTest(screenshare); + fixture->RunWithAnalyzer(screenshare); } -TEST_F(FullStackTest, ScreenshareSlidesVP8_3TL_Simulcast) { +TEST(FullStackTest, ScreenshareSlidesVP8_3TL_Simulcast) { + auto fixture = CreateVideoQualityTestFixture(); test::ScopedFieldTrials field_trial(kScreenshareSimulcastExperiment); - VideoQualityTest::Params screenshare; + ParamsWithLogging screenshare; screenshare.call.send_side_bwe = true; screenshare.screenshare[0] = {true, false, 10}; screenshare.video[0] = {true, 1850, 1110, 5, 800000, @@ -554,7 +595,7 @@ TEST_F(FullStackTest, ScreenshareSlidesVP8_3TL_Simulcast) { 2, 400000, false, false, false, ""}; screenshare.analyzer = {"screenshare_slides_simulcast", 0.0, 0.0, kFullStackTestDurationSecs}; - VideoQualityTest::Params screenshare_params_high; + ParamsWithLogging screenshare_params_high; screenshare_params_high.video[0] = {true, 1850, 1110, 5, 800000, 2500000, 2500000, false, "VP8", 3, 0, 400000, false, false, false, @@ -565,16 +606,17 @@ TEST_F(FullStackTest, ScreenshareSlidesVP8_3TL_Simulcast) { 0, 400000, false, false, false, ""}; std::vector streams = { - DefaultVideoStream(screenshare_params_low, 0), - DefaultVideoStream(screenshare_params_high, 0)}; + VideoQualityTest::DefaultVideoStream(screenshare_params_low, 0), + VideoQualityTest::DefaultVideoStream(screenshare_params_high, 0)}; screenshare.ss[0] = { streams, 1, 1, 0, InterLayerPredMode::kOn, std::vector(), false}; - RunTest(screenshare); + fixture->RunWithAnalyzer(screenshare); } -TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_Scroll) { - VideoQualityTest::Params config; +TEST(FullStackTest, ScreenshareSlidesVP8_2TL_Scroll) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging config; config.call.send_side_bwe = true; config.video[0] = {true, 1850, 1110 / 2, 5, 50000, 200000, 2000000, false, "VP8", 2, @@ -582,11 +624,12 @@ TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_Scroll) { config.screenshare[0] = {true, false, 10, 2}; config.analyzer = {"screenshare_slides_scrolling", 0.0, 0.0, kFullStackTestDurationSecs}; - RunTest(config); + fixture->RunWithAnalyzer(config); } -TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_LossyNet) { - VideoQualityTest::Params screenshare; +TEST(FullStackTest, ScreenshareSlidesVP8_2TL_LossyNet) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging screenshare; screenshare.call.send_side_bwe = true; screenshare.video[0] = {true, 1850, 1110, 5, 50000, 200000, 2000000, false, "VP8", 2, @@ -597,11 +640,12 @@ TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_LossyNet) { screenshare.pipe.loss_percent = 5; screenshare.pipe.queue_delay_ms = 200; screenshare.pipe.link_capacity_kbps = 500; - RunTest(screenshare); + fixture->RunWithAnalyzer(screenshare); } -TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_VeryLossyNet) { - VideoQualityTest::Params screenshare; +TEST(FullStackTest, ScreenshareSlidesVP8_2TL_VeryLossyNet) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging screenshare; screenshare.call.send_side_bwe = true; screenshare.video[0] = {true, 1850, 1110, 5, 50000, 200000, 2000000, false, "VP8", 2, @@ -612,11 +656,12 @@ TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_VeryLossyNet) { screenshare.pipe.loss_percent = 10; screenshare.pipe.queue_delay_ms = 200; screenshare.pipe.link_capacity_kbps = 500; - RunTest(screenshare); + fixture->RunWithAnalyzer(screenshare); } -TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_LossyNetRestrictedQueue) { - VideoQualityTest::Params screenshare; +TEST(FullStackTest, ScreenshareSlidesVP8_2TL_LossyNetRestrictedQueue) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging screenshare; screenshare.call.send_side_bwe = true; screenshare.video[0] = {true, 1850, 1110, 5, 50000, 200000, 2000000, false, "VP8", 2, @@ -628,11 +673,12 @@ TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_LossyNetRestrictedQueue) { screenshare.pipe.link_capacity_kbps = 200; screenshare.pipe.queue_length_packets = 30; - RunTest(screenshare); + fixture->RunWithAnalyzer(screenshare); } -TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_ModeratelyRestricted) { - VideoQualityTest::Params screenshare; +TEST(FullStackTest, ScreenshareSlidesVP8_2TL_ModeratelyRestricted) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging screenshare; screenshare.call.send_side_bwe = true; screenshare.video[0] = {true, 1850, 1110, 5, 50000, 200000, 2000000, false, "VP8", 2, @@ -644,13 +690,14 @@ TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_ModeratelyRestricted) { screenshare.pipe.link_capacity_kbps = 1200; screenshare.pipe.queue_length_packets = 30; - RunTest(screenshare); + fixture->RunWithAnalyzer(screenshare); } // TODO(sprang): Retire these tests once experiment is removed. -TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_LossyNetRestrictedQueue_ALR) { - test::ScopedFieldTrials field_trial(kAlrProbingExperiment); - VideoQualityTest::Params screenshare; +TEST(FullStackTest, ScreenshareSlidesVP8_2TL_LossyNetRestrictedQueue_ALR) { + auto fixture = CreateVideoQualityTestFixture(); + test::ScopedFieldTrials field_trial(AlrProbingExperimentName()); + ParamsWithLogging screenshare; screenshare.call.send_side_bwe = true; screenshare.video[0] = {true, 1850, 1110, 5, 50000, 200000, 2000000, false, "VP8", 2, @@ -662,12 +709,13 @@ TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_LossyNetRestrictedQueue_ALR) { screenshare.pipe.link_capacity_kbps = 200; screenshare.pipe.queue_length_packets = 30; - RunTest(screenshare); + fixture->RunWithAnalyzer(screenshare); } -TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_ALR) { - test::ScopedFieldTrials field_trial(kAlrProbingExperiment); - VideoQualityTest::Params screenshare; +TEST(FullStackTest, ScreenshareSlidesVP8_2TL_ALR) { + auto fixture = CreateVideoQualityTestFixture(); + test::ScopedFieldTrials field_trial(AlrProbingExperimentName()); + ParamsWithLogging screenshare; screenshare.call.send_side_bwe = true; screenshare.video[0] = {true, 1850, 1110, 5, 50000, 200000, 2000000, false, "VP8", 2, @@ -675,12 +723,13 @@ TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_ALR) { screenshare.screenshare[0] = {true, false, 10}; screenshare.analyzer = {"screenshare_slides_ALR", 0.0, 0.0, kFullStackTestDurationSecs}; - RunTest(screenshare); + fixture->RunWithAnalyzer(screenshare); } -TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_ModeratelyRestricted_ALR) { - test::ScopedFieldTrials field_trial(kAlrProbingExperiment); - VideoQualityTest::Params screenshare; +TEST(FullStackTest, ScreenshareSlidesVP8_2TL_ModeratelyRestricted_ALR) { + auto fixture = CreateVideoQualityTestFixture(); + test::ScopedFieldTrials field_trial(AlrProbingExperimentName()); + ParamsWithLogging screenshare; screenshare.call.send_side_bwe = true; screenshare.video[0] = {true, 1850, 1110, 5, 50000, 200000, 2000000, false, "VP8", 2, @@ -692,13 +741,15 @@ TEST_F(FullStackTest, ScreenshareSlidesVP8_2TL_ModeratelyRestricted_ALR) { screenshare.pipe.link_capacity_kbps = 1200; screenshare.pipe.queue_length_packets = 30; - RunTest(screenshare); + fixture->RunWithAnalyzer(screenshare); } -TEST_F(FullStackTest, ScreenshareSlidesVP8_3TL_Simulcast_ALR) { - test::ScopedFieldTrials field_trial(kScreenshareSimulcastExperiment + - kAlrProbingExperiment); - VideoQualityTest::Params screenshare; +TEST(FullStackTest, ScreenshareSlidesVP8_3TL_Simulcast_ALR) { + auto fixture = CreateVideoQualityTestFixture(); + test::ScopedFieldTrials field_trial( + std::string(kScreenshareSimulcastExperiment) + + AlrProbingExperimentName()); + ParamsWithLogging screenshare; screenshare.call.send_side_bwe = true; screenshare.screenshare[0] = {true, false, 10}; screenshare.video[0] = {true, 1850, 1110, 5, 800000, @@ -706,7 +757,7 @@ TEST_F(FullStackTest, ScreenshareSlidesVP8_3TL_Simulcast_ALR) { 2, 400000, false, false, false, ""}; screenshare.analyzer = {"screenshare_slides_simulcast_alr", 0.0, 0.0, kFullStackTestDurationSecs}; - VideoQualityTest::Params screenshare_params_high; + ParamsWithLogging screenshare_params_high; screenshare_params_high.video[0] = {true, 1850, 1110, 5, 800000, 2500000, 2500000, false, "VP8", 3, 0, 400000, false, false, false, @@ -717,37 +768,38 @@ TEST_F(FullStackTest, ScreenshareSlidesVP8_3TL_Simulcast_ALR) { 0, 400000, false, false, false, ""}; std::vector streams = { - DefaultVideoStream(screenshare_params_low, 0), - DefaultVideoStream(screenshare_params_high, 0)}; + VideoQualityTest::DefaultVideoStream(screenshare_params_low, 0), + VideoQualityTest::DefaultVideoStream(screenshare_params_high, 0)}; screenshare.ss[0] = { streams, 1, 1, 0, InterLayerPredMode::kOn, std::vector(), false}; - RunTest(screenshare); + fixture->RunWithAnalyzer(screenshare); } -const VideoQualityTest::Params::Video kSvcVp9Video = { +const ParamsWithLogging::Video kSvcVp9Video = { true, 1280, 720, 30, 800000, 2500000, 2500000, false, "VP9", 3, 2, 400000, false, false, false, "ConferenceMotion_1280_720_50"}; -const VideoQualityTest::Params::Video kSimulcastVp8VideoHigh = { +const ParamsWithLogging::Video kSimulcastVp8VideoHigh = { true, 1280, 720, 30, 800000, 2500000, 2500000, false, "VP8", 3, 2, 400000, false, false, false, "ConferenceMotion_1280_720_50"}; -const VideoQualityTest::Params::Video kSimulcastVp8VideoMedium = { +const ParamsWithLogging::Video kSimulcastVp8VideoMedium = { true, 640, 360, 30, 150000, 500000, 700000, false, "VP8", 3, 2, 400000, false, false, false, "ConferenceMotion_1280_720_50"}; -const VideoQualityTest::Params::Video kSimulcastVp8VideoLow = { +const ParamsWithLogging::Video kSimulcastVp8VideoLow = { true, 320, 180, 30, 30000, 150000, 200000, false, "VP8", 3, 2, 400000, false, false, false, "ConferenceMotion_1280_720_50"}; #if !defined(RTC_DISABLE_VP9) -TEST_F(FullStackTest, ScreenshareSlidesVP9_2SL) { - VideoQualityTest::Params screenshare; +TEST(FullStackTest, ScreenshareSlidesVP9_2SL) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging screenshare; screenshare.call.send_side_bwe = true; screenshare.video[0] = {true, 1850, 1110, 5, 50000, 200000, 2000000, false, "VP9", 1, @@ -758,23 +810,26 @@ TEST_F(FullStackTest, ScreenshareSlidesVP9_2SL) { screenshare.ss[0] = { std::vector(), 0, 2, 1, InterLayerPredMode::kOn, std::vector(), false}; - RunTest(screenshare); + fixture->RunWithAnalyzer(screenshare); } -TEST_F(FullStackTest, VP9SVC_3SL_High) { - VideoQualityTest::Params simulcast; +TEST(FullStackTest, VP9SVC_3SL_High) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging simulcast; simulcast.call.send_side_bwe = true; simulcast.video[0] = kSvcVp9Video; simulcast.analyzer = {"vp9svc_3sl_high", 0.0, 0.0, kFullStackTestDurationSecs}; + simulcast.ss[0] = { std::vector(), 0, 3, 2, InterLayerPredMode::kOn, std::vector(), false}; - RunTest(simulcast); + fixture->RunWithAnalyzer(simulcast); } -TEST_F(FullStackTest, VP9SVC_3SL_Medium) { - VideoQualityTest::Params simulcast; +TEST(FullStackTest, VP9SVC_3SL_Medium) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging simulcast; simulcast.call.send_side_bwe = true; simulcast.video[0] = kSvcVp9Video; simulcast.analyzer = {"vp9svc_3sl_medium", 0.0, 0.0, @@ -782,22 +837,24 @@ TEST_F(FullStackTest, VP9SVC_3SL_Medium) { simulcast.ss[0] = { std::vector(), 0, 3, 1, InterLayerPredMode::kOn, std::vector(), false}; - RunTest(simulcast); + fixture->RunWithAnalyzer(simulcast); } -TEST_F(FullStackTest, VP9SVC_3SL_Low) { - VideoQualityTest::Params simulcast; +TEST(FullStackTest, VP9SVC_3SL_Low) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging simulcast; simulcast.call.send_side_bwe = true; simulcast.video[0] = kSvcVp9Video; simulcast.analyzer = {"vp9svc_3sl_low", 0.0, 0.0, kFullStackTestDurationSecs}; simulcast.ss[0] = { std::vector(), 0, 3, 0, InterLayerPredMode::kOn, std::vector(), false}; - RunTest(simulcast); + fixture->RunWithAnalyzer(simulcast); } -TEST_F(FullStackTest, VP9KSVC_3SL_High) { - VideoQualityTest::Params simulcast; +TEST(FullStackTest, VP9KSVC_3SL_High) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging simulcast; simulcast.call.send_side_bwe = true; simulcast.video[0] = kSvcVp9Video; simulcast.analyzer = {"vp9ksvc_3sl_high", 0.0, 0.0, @@ -805,11 +862,12 @@ TEST_F(FullStackTest, VP9KSVC_3SL_High) { simulcast.ss[0] = { std::vector(), 0, 3, 2, InterLayerPredMode::kOnKeyPic, std::vector(), false}; - RunTest(simulcast); + fixture->RunWithAnalyzer(simulcast); } -TEST_F(FullStackTest, VP9KSVC_3SL_Medium) { - VideoQualityTest::Params simulcast; +TEST(FullStackTest, VP9KSVC_3SL_Medium) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging simulcast; simulcast.call.send_side_bwe = true; simulcast.video[0] = kSvcVp9Video; simulcast.analyzer = {"vp9ksvc_3sl_medium", 0.0, 0.0, @@ -817,11 +875,12 @@ TEST_F(FullStackTest, VP9KSVC_3SL_Medium) { simulcast.ss[0] = { std::vector(), 0, 3, 1, InterLayerPredMode::kOnKeyPic, std::vector(), false}; - RunTest(simulcast); + fixture->RunWithAnalyzer(simulcast); } -TEST_F(FullStackTest, VP9KSVC_3SL_Low) { - VideoQualityTest::Params simulcast; +TEST(FullStackTest, VP9KSVC_3SL_Low) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging simulcast; simulcast.call.send_side_bwe = true; simulcast.video[0] = kSvcVp9Video; simulcast.analyzer = {"vp9ksvc_3sl_low", 0.0, 0.0, @@ -829,10 +888,11 @@ TEST_F(FullStackTest, VP9KSVC_3SL_Low) { simulcast.ss[0] = { std::vector(), 0, 3, 0, InterLayerPredMode::kOnKeyPic, std::vector(), false}; - RunTest(simulcast); + fixture->RunWithAnalyzer(simulcast); } -TEST_F(FullStackTest, VP9KSVC_3SL_Medium_Network_Restricted) { +TEST(FullStackTest, VP9KSVC_3SL_Medium_Network_Restricted) { + auto fixture = CreateVideoQualityTestFixture(); VideoQualityTest::Params simulcast; simulcast.call.send_side_bwe = true; simulcast.video[0] = kSvcVp9Video; @@ -842,7 +902,7 @@ TEST_F(FullStackTest, VP9KSVC_3SL_Medium_Network_Restricted) { std::vector(), 0, 3, 1, InterLayerPredMode::kOnKeyPic, std::vector(), false}; simulcast.pipe.link_capacity_kbps = 1000; - RunTest(simulcast); + fixture->RunWithAnalyzer(simulcast); } #endif // !defined(RTC_DISABLE_VP9) @@ -855,8 +915,9 @@ TEST_F(FullStackTest, VP9KSVC_3SL_Medium_Network_Restricted) { #define MAYBE_SimulcastFullHdOveruse SimulcastFullHdOveruse #endif -TEST_F(FullStackTest, MAYBE_SimulcastFullHdOveruse) { - VideoQualityTest::Params simulcast; +TEST(FullStackTest, MAYBE_SimulcastFullHdOveruse) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging simulcast; simulcast.call.send_side_bwe = true; simulcast.video[0] = {true, 1920, 1080, 30, 800000, 2500000, 2500000, false, "VP8", 3, @@ -865,116 +926,122 @@ TEST_F(FullStackTest, MAYBE_SimulcastFullHdOveruse) { kFullStackTestDurationSecs}; simulcast.pipe.loss_percent = 0; simulcast.pipe.queue_delay_ms = 100; - std::vector streams = {DefaultVideoStream(simulcast, 0), - DefaultVideoStream(simulcast, 0), - DefaultVideoStream(simulcast, 0)}; + std::vector streams = { + VideoQualityTest::DefaultVideoStream(simulcast, 0), + VideoQualityTest::DefaultVideoStream(simulcast, 0), + VideoQualityTest::DefaultVideoStream(simulcast, 0) + }; simulcast.ss[0] = { streams, 2, 1, 0, InterLayerPredMode::kOn, std::vector(), true}; webrtc::test::ScopedFieldTrials override_trials( "WebRTC-ForceSimulatedOveruseIntervalMs/1000-50000-300/"); - RunTest(simulcast); + fixture->RunWithAnalyzer(simulcast); } -TEST_F(FullStackTest, SimulcastVP8_3SL_High) { - VideoQualityTest::Params simulcast; +TEST(FullStackTest, SimulcastVP8_3SL_High) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging simulcast; simulcast.call.send_side_bwe = true; simulcast.video[0] = kSimulcastVp8VideoHigh; simulcast.analyzer = {"simulcast_vp8_3sl_high", 0.0, 0.0, kFullStackTestDurationSecs}; simulcast.pipe.loss_percent = 0; simulcast.pipe.queue_delay_ms = 100; - VideoQualityTest::Params video_params_high; + ParamsWithLogging video_params_high; video_params_high.video[0] = kSimulcastVp8VideoHigh; - VideoQualityTest::Params video_params_medium; + ParamsWithLogging video_params_medium; video_params_medium.video[0] = kSimulcastVp8VideoMedium; - VideoQualityTest::Params video_params_low; + ParamsWithLogging video_params_low; video_params_low.video[0] = kSimulcastVp8VideoLow; std::vector streams = { - DefaultVideoStream(video_params_low, 0), - DefaultVideoStream(video_params_medium, 0), - DefaultVideoStream(video_params_high, 0)}; + VideoQualityTest::DefaultVideoStream(video_params_low, 0), + VideoQualityTest::DefaultVideoStream(video_params_medium, 0), + VideoQualityTest::DefaultVideoStream(video_params_high, 0)}; simulcast.ss[0] = { streams, 2, 1, 0, InterLayerPredMode::kOn, std::vector(), false}; - RunTest(simulcast); + fixture->RunWithAnalyzer(simulcast); } -TEST_F(FullStackTest, SimulcastVP8_3SL_Medium) { - VideoQualityTest::Params simulcast; +TEST(FullStackTest, SimulcastVP8_3SL_Medium) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging simulcast; simulcast.call.send_side_bwe = true; simulcast.video[0] = kSimulcastVp8VideoHigh; simulcast.analyzer = {"simulcast_vp8_3sl_medium", 0.0, 0.0, kFullStackTestDurationSecs}; simulcast.pipe.loss_percent = 0; simulcast.pipe.queue_delay_ms = 100; - VideoQualityTest::Params video_params_high; + ParamsWithLogging video_params_high; video_params_high.video[0] = kSimulcastVp8VideoHigh; - VideoQualityTest::Params video_params_medium; + ParamsWithLogging video_params_medium; video_params_medium.video[0] = kSimulcastVp8VideoMedium; - VideoQualityTest::Params video_params_low; + ParamsWithLogging video_params_low; video_params_low.video[0] = kSimulcastVp8VideoLow; std::vector streams = { - DefaultVideoStream(video_params_low, 0), - DefaultVideoStream(video_params_medium, 0), - DefaultVideoStream(video_params_high, 0)}; + VideoQualityTest::DefaultVideoStream(video_params_low, 0), + VideoQualityTest::DefaultVideoStream(video_params_medium, 0), + VideoQualityTest::DefaultVideoStream(video_params_high, 0)}; simulcast.ss[0] = { streams, 1, 1, 0, InterLayerPredMode::kOn, std::vector(), false}; - RunTest(simulcast); + fixture->RunWithAnalyzer(simulcast); } -TEST_F(FullStackTest, SimulcastVP8_3SL_Low) { - VideoQualityTest::Params simulcast; +TEST(FullStackTest, SimulcastVP8_3SL_Low) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging simulcast; simulcast.call.send_side_bwe = true; simulcast.video[0] = kSimulcastVp8VideoHigh; simulcast.analyzer = {"simulcast_vp8_3sl_low", 0.0, 0.0, kFullStackTestDurationSecs}; simulcast.pipe.loss_percent = 0; simulcast.pipe.queue_delay_ms = 100; - VideoQualityTest::Params video_params_high; + ParamsWithLogging video_params_high; video_params_high.video[0] = kSimulcastVp8VideoHigh; - VideoQualityTest::Params video_params_medium; + ParamsWithLogging video_params_medium; video_params_medium.video[0] = kSimulcastVp8VideoMedium; - VideoQualityTest::Params video_params_low; + ParamsWithLogging video_params_low; video_params_low.video[0] = kSimulcastVp8VideoLow; std::vector streams = { - DefaultVideoStream(video_params_low, 0), - DefaultVideoStream(video_params_medium, 0), - DefaultVideoStream(video_params_high, 0)}; + VideoQualityTest::DefaultVideoStream(video_params_low, 0), + VideoQualityTest::DefaultVideoStream(video_params_medium, 0), + VideoQualityTest::DefaultVideoStream(video_params_high, 0)}; simulcast.ss[0] = { streams, 0, 1, 0, InterLayerPredMode::kOn, std::vector(), false}; - RunTest(simulcast); + fixture->RunWithAnalyzer(simulcast); } -TEST_F(FullStackTest, LargeRoomVP8_5thumb) { - VideoQualityTest::Params large_room; +TEST(FullStackTest, LargeRoomVP8_5thumb) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging large_room; large_room.call.send_side_bwe = true; large_room.video[0] = kSimulcastVp8VideoHigh; large_room.analyzer = {"largeroom_5thumb", 0.0, 0.0, kFullStackTestDurationSecs}; large_room.pipe.loss_percent = 0; large_room.pipe.queue_delay_ms = 100; - VideoQualityTest::Params video_params_high; + ParamsWithLogging video_params_high; video_params_high.video[0] = kSimulcastVp8VideoHigh; - VideoQualityTest::Params video_params_medium; + ParamsWithLogging video_params_medium; video_params_medium.video[0] = kSimulcastVp8VideoMedium; - VideoQualityTest::Params video_params_low; + ParamsWithLogging video_params_low; video_params_low.video[0] = kSimulcastVp8VideoLow; std::vector streams = { - DefaultVideoStream(video_params_low, 0), - DefaultVideoStream(video_params_medium, 0), - DefaultVideoStream(video_params_high, 0)}; + VideoQualityTest::DefaultVideoStream(video_params_low, 0), + VideoQualityTest::DefaultVideoStream(video_params_medium, 0), + VideoQualityTest::DefaultVideoStream(video_params_high, 0)}; large_room.call.num_thumbnails = 5; large_room.ss[0] = { streams, 2, 1, 0, InterLayerPredMode::kOn, std::vector(), false}; - RunTest(large_room); + fixture->RunWithAnalyzer(large_room); } #if defined(WEBRTC_ANDROID) || defined(WEBRTC_IOS) @@ -987,70 +1054,73 @@ TEST_F(FullStackTest, LargeRoomVP8_5thumb) { #define MAYBE_LargeRoomVP8_15thumb LargeRoomVP8_15thumb #endif -TEST_F(FullStackTest, MAYBE_LargeRoomVP8_15thumb) { - VideoQualityTest::Params large_room; +TEST(FullStackTest, MAYBE_LargeRoomVP8_15thumb) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging large_room; large_room.call.send_side_bwe = true; large_room.video[0] = kSimulcastVp8VideoHigh; large_room.analyzer = {"largeroom_15thumb", 0.0, 0.0, kFullStackTestDurationSecs}; large_room.pipe.loss_percent = 0; large_room.pipe.queue_delay_ms = 100; - VideoQualityTest::Params video_params_high; + ParamsWithLogging video_params_high; video_params_high.video[0] = kSimulcastVp8VideoHigh; - VideoQualityTest::Params video_params_medium; + ParamsWithLogging video_params_medium; video_params_medium.video[0] = kSimulcastVp8VideoMedium; - VideoQualityTest::Params video_params_low; + ParamsWithLogging video_params_low; video_params_low.video[0] = kSimulcastVp8VideoLow; std::vector streams = { - DefaultVideoStream(video_params_low, 0), - DefaultVideoStream(video_params_medium, 0), - DefaultVideoStream(video_params_high, 0)}; + VideoQualityTest::DefaultVideoStream(video_params_low, 0), + VideoQualityTest::DefaultVideoStream(video_params_medium, 0), + VideoQualityTest::DefaultVideoStream(video_params_high, 0)}; large_room.call.num_thumbnails = 15; large_room.ss[0] = { streams, 2, 1, 0, InterLayerPredMode::kOn, std::vector(), false}; - RunTest(large_room); + fixture->RunWithAnalyzer(large_room); } -TEST_F(FullStackTest, MAYBE_LargeRoomVP8_50thumb) { - VideoQualityTest::Params large_room; +TEST(FullStackTest, MAYBE_LargeRoomVP8_50thumb) { + auto fixture = CreateVideoQualityTestFixture(); + ParamsWithLogging large_room; large_room.call.send_side_bwe = true; large_room.video[0] = kSimulcastVp8VideoHigh; large_room.analyzer = {"largeroom_50thumb", 0.0, 0.0, kFullStackTestDurationSecs}; large_room.pipe.loss_percent = 0; large_room.pipe.queue_delay_ms = 100; - VideoQualityTest::Params video_params_high; + ParamsWithLogging video_params_high; video_params_high.video[0] = kSimulcastVp8VideoHigh; - VideoQualityTest::Params video_params_medium; + ParamsWithLogging video_params_medium; video_params_medium.video[0] = kSimulcastVp8VideoMedium; - VideoQualityTest::Params video_params_low; + ParamsWithLogging video_params_low; video_params_low.video[0] = kSimulcastVp8VideoLow; std::vector streams = { - DefaultVideoStream(video_params_low, 0), - DefaultVideoStream(video_params_medium, 0), - DefaultVideoStream(video_params_high, 0)}; + VideoQualityTest::DefaultVideoStream(video_params_low, 0), + VideoQualityTest::DefaultVideoStream(video_params_medium, 0), + VideoQualityTest::DefaultVideoStream(video_params_high, 0)}; large_room.call.num_thumbnails = 50; large_room.ss[0] = { streams, 2, 1, 0, InterLayerPredMode::kOn, std::vector(), false}; - RunTest(large_room); + fixture->RunWithAnalyzer(large_room); } -class DualStreamsTest : public FullStackTest, - public ::testing::WithParamInterface {}; +class DualStreamsTest : public ::testing::TestWithParam {}; // Disable dual video test on mobile device becuase it's too heavy. #if !defined(WEBRTC_ANDROID) && !defined(WEBRTC_IOS) TEST_P(DualStreamsTest, ModeratelyRestricted_SlidesVp8_3TL_Simulcast_Video_Simulcast_High) { test::ScopedFieldTrials field_trial( - kScreenshareSimulcastExperiment + kAlrProbingExperiment + - kRoundRobinPacingQueueExperiment + kPacerPushBackExperiment); + std::string(kScreenshareSimulcastExperiment) + + AlrProbingExperimentName() + + std::string(kRoundRobinPacingQueueExperiment) + + std::string(kPacerPushBackExperiment)); const int first_stream = GetParam(); - VideoQualityTest::Params dual_streams; + ParamsWithLogging dual_streams; // Screenshare Settings. dual_streams.screenshare[first_stream] = {true, false, 10}; @@ -1059,7 +1129,7 @@ TEST_P(DualStreamsTest, 2, 400000, false, false, false, ""}; - VideoQualityTest::Params screenshare_params_high; + ParamsWithLogging screenshare_params_high; screenshare_params_high.video[0] = {true, 1850, 1110, 5, 800000, 2500000, 2500000, false, "VP8", 3, 0, 400000, false, false, false, @@ -1069,8 +1139,8 @@ TEST_P(DualStreamsTest, 200000, 2000000, false, "VP8", 2, 0, 400000, false, false, false, ""}; std::vector screenhsare_streams = { - DefaultVideoStream(screenshare_params_low, 0), - DefaultVideoStream(screenshare_params_high, 0)}; + VideoQualityTest::DefaultVideoStream(screenshare_params_low, 0), + VideoQualityTest::DefaultVideoStream(screenshare_params_high, 0)}; dual_streams.ss[first_stream] = { screenhsare_streams, 1, 1, 0, InterLayerPredMode::kOn, @@ -1079,16 +1149,16 @@ TEST_P(DualStreamsTest, // Video settings. dual_streams.video[1 - first_stream] = kSimulcastVp8VideoHigh; - VideoQualityTest::Params video_params_high; + ParamsWithLogging video_params_high; video_params_high.video[0] = kSimulcastVp8VideoHigh; - VideoQualityTest::Params video_params_medium; + ParamsWithLogging video_params_medium; video_params_medium.video[0] = kSimulcastVp8VideoMedium; - VideoQualityTest::Params video_params_low; + ParamsWithLogging video_params_low; video_params_low.video[0] = kSimulcastVp8VideoLow; std::vector streams = { - DefaultVideoStream(video_params_low, 0), - DefaultVideoStream(video_params_medium, 0), - DefaultVideoStream(video_params_high, 0)}; + VideoQualityTest::DefaultVideoStream(video_params_low, 0), + VideoQualityTest::DefaultVideoStream(video_params_medium, 0), + VideoQualityTest::DefaultVideoStream(video_params_high, 0)}; dual_streams.ss[1 - first_stream] = { streams, 2, 1, 0, InterLayerPredMode::kOn, std::vector(), @@ -1105,15 +1175,17 @@ TEST_P(DualStreamsTest, dual_streams.pipe.queue_length_packets = 30; dual_streams.pipe.queue_delay_ms = 100; - RunTest(dual_streams); + auto fixture = CreateVideoQualityTestFixture(); + fixture->RunWithAnalyzer(dual_streams); } #endif // !defined(WEBRTC_ANDROID) && !defined(WEBRTC_IOS) TEST_P(DualStreamsTest, Conference_Restricted) { - test::ScopedFieldTrials field_trial(kRoundRobinPacingQueueExperiment + - kPacerPushBackExperiment); + test::ScopedFieldTrials field_trial( + std::string(kRoundRobinPacingQueueExperiment) + + std::string(kPacerPushBackExperiment)); const int first_stream = GetParam(); - VideoQualityTest::Params dual_streams; + ParamsWithLogging dual_streams; // Screenshare Settings. dual_streams.screenshare[first_stream] = {true, false, 10}; @@ -1138,7 +1210,8 @@ TEST_P(DualStreamsTest, Conference_Restricted) { dual_streams.pipe.queue_length_packets = 30; dual_streams.pipe.queue_delay_ms = 100; - RunTest(dual_streams); + auto fixture = CreateVideoQualityTestFixture(); + fixture->RunWithAnalyzer(dual_streams); } INSTANTIATE_TEST_CASE_P(FullStackTest, diff --git a/video/screenshare_loopback.cc b/video/screenshare_loopback.cc index 81bc34697d..19106d313c 100644 --- a/video/screenshare_loopback.cc +++ b/video/screenshare_loopback.cc @@ -333,11 +333,11 @@ void Loopback() { flags::SelectedStream(), flags::NumSpatialLayers(), flags::SelectedSL(), flags::InterLayerPred(), SL_descriptors); - VideoQualityTest test; + auto fixture = rtc::MakeUnique(nullptr); if (flags::DurationSecs()) { - test.RunWithAnalyzer(params); + fixture->RunWithAnalyzer(params); } else { - test.RunWithRenderers(params); + fixture->RunWithRenderers(params); } } } // namespace webrtc diff --git a/video/sv_loopback.cc b/video/sv_loopback.cc index 9b0f71b822..9c336105e3 100644 --- a/video/sv_loopback.cc +++ b/video/sv_loopback.cc @@ -578,11 +578,11 @@ void Loopback() { flags::VideoSelectedStream(), flags::VideoNumSpatialLayers(), flags::VideoSelectedSL(), flags::VideoInterLayerPred(), SL_descriptors); - VideoQualityTest test; + auto fixture = rtc::MakeUnique(nullptr); if (flags::DurationSecs()) { - test.RunWithAnalyzer(params); + fixture->RunWithAnalyzer(params); } else { - test.RunWithRenderers(params); + fixture->RunWithRenderers(params); } } } // namespace webrtc diff --git a/video/video_loopback.cc b/video/video_loopback.cc index 6405fc8646..2f6d66d28b 100644 --- a/video/video_loopback.cc +++ b/video/video_loopback.cc @@ -334,11 +334,11 @@ void Loopback() { flags::SelectedStream(), flags::NumSpatialLayers(), flags::SelectedSL(), flags::InterLayerPred(), SL_descriptors); - VideoQualityTest test; + auto fixture = rtc::MakeUnique(nullptr); if (flags::DurationSecs()) { - test.RunWithAnalyzer(params); + fixture->RunWithAnalyzer(params); } else { - test.RunWithRenderers(params); + fixture->RunWithRenderers(params); } } } // namespace webrtc diff --git a/video/video_quality_test.cc b/video/video_quality_test.cc index c12212e35a..cbc89a331d 100644 --- a/video/video_quality_test.cc +++ b/video/video_quality_test.cc @@ -1089,7 +1089,8 @@ VideoQualityTest::TestVideoEncoderFactory::CreateVideoEncoder( } } -VideoQualityTest::VideoQualityTest() +VideoQualityTest::VideoQualityTest( + std::unique_ptr fec_controller_factory) : clock_(Clock::GetRealTimeClock()), receive_logs_(0), send_logs_(0) { payload_type_map_ = test::CallTest::payload_type_map_; RTC_DCHECK(payload_type_map_.find(kPayloadTypeH264) == @@ -1101,11 +1102,7 @@ VideoQualityTest::VideoQualityTest() payload_type_map_[kPayloadTypeH264] = webrtc::MediaType::VIDEO; payload_type_map_[kPayloadTypeVP8] = webrtc::MediaType::VIDEO; payload_type_map_[kPayloadTypeVP9] = webrtc::MediaType::VIDEO; -} -VideoQualityTest::VideoQualityTest( - std::unique_ptr fec_controller_factory) - : VideoQualityTest() { fec_controller_factory_ = std::move(fec_controller_factory); } diff --git a/video/video_quality_test.h b/video/video_quality_test.h index 76c8f64090..7da67523fa 100644 --- a/video/video_quality_test.h +++ b/video/video_quality_test.h @@ -15,6 +15,9 @@ #include #include +#include "api/fec_controller.h" +#include "api/test/video_quality_test_fixture.h" +#include "call/fake_network_pipe.h" #include "media/engine/internalencoderfactory.h" #include "test/call_test.h" #include "test/frame_generator.h" @@ -22,86 +25,18 @@ namespace webrtc { -class VideoQualityTest : public test::CallTest { +class VideoQualityTest : + public test::CallTest, public VideoQualityTestFixtureInterface { public: - // Parameters are grouped into smaller structs to make it easier to set - // the desired elements and skip unused, using aggregate initialization. - // Unfortunately, C++11 (as opposed to C11) doesn't support unnamed structs, - // which makes the implementation of VideoQualityTest a bit uglier. - struct Params { - Params(); - ~Params(); - struct CallConfig { - bool send_side_bwe; - BitrateConstraints call_bitrate_config; - int num_thumbnails; - // Indicates if secondary_(video|ss|screenshare) structures are used. - bool dual_video; - } call; - struct Video { - bool enabled; - size_t width; - size_t height; - int32_t fps; - int min_bitrate_bps; - int target_bitrate_bps; - int max_bitrate_bps; - bool suspend_below_min_bitrate; - std::string codec; - int num_temporal_layers; - int selected_tl; - int min_transmit_bps; - bool ulpfec; - bool flexfec; - bool automatic_scaling; - std::string clip_name; // "Generator" to generate frames instead. - size_t capture_device_index; - } video[2]; - struct Audio { - bool enabled; - bool sync_video; - bool dtx; - } audio; - struct Screenshare { - bool enabled; - bool generate_slides; - int32_t slide_change_interval; - int32_t scroll_duration; - std::vector slides; - } screenshare[2]; - struct Analyzer { - std::string test_label; - double avg_psnr_threshold; // (*) - double avg_ssim_threshold; // (*) - int test_durations_secs; - std::string graph_data_output_filename; - std::string graph_title; - } analyzer; - FakeNetworkPipe::Config pipe; - struct SS { // Spatial scalability. - std::vector streams; // If empty, one stream is assumed. - size_t selected_stream; - int num_spatial_layers; - int selected_sl; - InterLayerPredMode inter_layer_pred; - // If empty, bitrates are generated in VP9Impl automatically. - std::vector spatial_layers; - // If set, default parameters will be used instead of |streams|. - bool infer_streams; - } ss[2]; - struct Logging { - bool logs; - std::string rtc_event_log_name; - std::string rtp_dump_name; - std::string encoded_frame_base_path; - } logging; - }; - - VideoQualityTest(); explicit VideoQualityTest( std::unique_ptr fec_controller_factory); - void RunWithAnalyzer(const Params& params); - void RunWithRenderers(const Params& params); + + void RunWithAnalyzer(const Params& params) override; + void RunWithRenderers(const Params& params) override; + + const std::map& payload_type_map() override { + return payload_type_map_; + } static void FillScalabilitySettings( Params* params, @@ -114,6 +49,11 @@ class VideoQualityTest : public test::CallTest { InterLayerPredMode inter_layer_pred, const std::vector& sl_descriptors); + // Helper static methods. + static VideoStream DefaultVideoStream(const Params& params, size_t video_idx); + static VideoStream DefaultThumbnailStream(); + static std::vector ParseCSV(const std::string& str); + protected: class TestVideoEncoderFactory : public VideoEncoderFactory { std::vector GetSupportedFormats() const override; @@ -138,11 +78,6 @@ class VideoQualityTest : public test::CallTest { std::string GenerateGraphTitle() const; void CheckParams(); - // Helper static methods. - static VideoStream DefaultVideoStream(const Params& params, size_t video_idx); - static VideoStream DefaultThumbnailStream(); - static std::vector ParseCSV(const std::string& str); - // Helper methods for setting up the call. void CreateVideoStreams(); void DestroyStreams();