diff --git a/src/modules/video_coding/codecs/tools/video_codecs_tools.gypi b/src/modules/video_coding/codecs/tools/video_codecs_tools.gypi index dc327fc0d4..a0d874eea8 100644 --- a/src/modules/video_coding/codecs/tools/video_codecs_tools.gypi +++ b/src/modules/video_coding/codecs/tools/video_codecs_tools.gypi @@ -16,7 +16,7 @@ 'type': 'executable', 'dependencies': [ 'video_codecs_test_framework', - 'video_coding_test_lib', + '<(webrtc_root)/../test/test.gyp:test_support', 'webrtc_vp8', '<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers', '<(webrtc_root)/common_video/common_video.gyp:webrtc_vplib', diff --git a/src/modules/video_coding/codecs/tools/video_quality_measurement.cc b/src/modules/video_coding/codecs/tools/video_quality_measurement.cc index c85353bf0c..95e54387a7 100644 --- a/src/modules/video_coding/codecs/tools/video_quality_measurement.cc +++ b/src/modules/video_coding/codecs/tools/video_quality_measurement.cc @@ -21,8 +21,8 @@ #include "packet_reader.h" #include "stats.h" #include "trace.h" +#include "testsupport/metrics/video_metrics.h" #include "util.h" -#include "video_metrics.h" #include "videoprocessor.h" #include "vp8.h" @@ -105,7 +105,7 @@ int HandleCommandLineFlags(webrtc::test::TestConfig* config) { // Verify the output dir exists: struct stat dir_info; - if (!(stat(FLAGS_output_dir.c_str(), &dir_info) == 0 && + if (!(stat(FLAGS_output_dir.c_str(), &dir_info) == 0 && S_ISDIR(dir_info.st_mode))) { fprintf(stderr, "Cannot find output directory: %s\n", FLAGS_output_dir.c_str()); diff --git a/src/modules/video_coding/main/source/video_coding_test.gypi b/src/modules/video_coding/main/source/video_coding_test.gypi index 7c0f78f74f..57c34d0b92 100644 --- a/src/modules/video_coding/main/source/video_coding_test.gypi +++ b/src/modules/video_coding/main/source/video_coding_test.gypi @@ -7,27 +7,11 @@ # be found in the AUTHORS file in the root of the source tree. { - 'targets': [ - { - 'target_name': 'video_coding_test_lib', - 'type': '<(library)', - 'direct_dependent_settings': { - 'include_dirs': [ - '../test', - ], - }, - 'sources': [ - # headers - '../test/video_metrics.h', - # sources - '../test/video_metrics.cc', - ], - }, - { + 'targets': [{ 'target_name': 'video_coding_test', 'type': 'executable', 'dependencies': [ - 'video_coding_test_lib', + '<(webrtc_root)/../test/test.gyp:test_support', 'webrtc_video_coding', 'rtp_rtcp', 'webrtc_utility', diff --git a/src/modules/video_coding/main/test/codec_database_test.cc b/src/modules/video_coding/main/test/codec_database_test.cc index 34d120d9ce..56b022a38c 100644 --- a/src/modules/video_coding/main/test/codec_database_test.cc +++ b/src/modules/video_coding/main/test/codec_database_test.cc @@ -21,7 +21,7 @@ #include "test_callbacks.h" #include "test_macros.h" #include "test_util.h" -#include "video_metrics.h" +#include "testsupport/metrics/video_metrics.h" #include "vp8.h" // for external codecs test diff --git a/src/modules/video_coding/main/test/media_opt_test.cc b/src/modules/video_coding/main/test/media_opt_test.cc index 5ec05efdbd..36832dc61d 100644 --- a/src/modules/video_coding/main/test/media_opt_test.cc +++ b/src/modules/video_coding/main/test/media_opt_test.cc @@ -23,8 +23,8 @@ #include "test_callbacks.h" #include "test_macros.h" #include "test_util.h" // send side callback +#include "testsupport/metrics/video_metrics.h" #include "video_coding.h" -#include "video_metrics.h" using namespace webrtc; diff --git a/src/modules/video_coding/main/test/normal_test.cc b/src/modules/video_coding/main/test/normal_test.cc index 0d3ed51abc..76d61e9131 100644 --- a/src/modules/video_coding/main/test/normal_test.cc +++ b/src/modules/video_coding/main/test/normal_test.cc @@ -22,8 +22,7 @@ #include "test_util.h" #include "tick_time.h" #include "trace.h" -#include "video_metrics.h" - +#include "testsupport/metrics/video_metrics.h" using namespace webrtc; diff --git a/src/modules/video_coding/main/test/quality_modes_test.cc b/src/modules/video_coding/main/test/quality_modes_test.cc index c8215daf31..f39c786bf8 100644 --- a/src/modules/video_coding/main/test/quality_modes_test.cc +++ b/src/modules/video_coding/main/test/quality_modes_test.cc @@ -17,7 +17,7 @@ #include "../source/event.h" #include "test_callbacks.h" #include "test_macros.h" -#include "video_metrics.h" +#include "testsupport/metrics/video_metrics.h" #include "vplib.h" using namespace webrtc; diff --git a/src/video_engine/main/test/AutoTest/automated/vie_standard_integration_test.cc b/src/video_engine/main/test/AutoTest/automated/vie_standard_integration_test.cc index 1b556d4846..5cbe2e4f94 100644 --- a/src/video_engine/main/test/AutoTest/automated/vie_standard_integration_test.cc +++ b/src/video_engine/main/test/AutoTest/automated/vie_standard_integration_test.cc @@ -15,22 +15,30 @@ * to verify any video output - it only checks for direct errors. */ +#include + #include "gflags/gflags.h" #include "gtest/gtest.h" +#include "testsupport/metrics/video_metrics.h" #include "vie_autotest.h" #include "vie_autotest_window_manager_interface.h" #include "vie_integration_test_base.h" +#include "vie_to_file_renderer.h" #include "vie_window_creator.h" namespace { +// We limit the input video size in order to not use to much bandwidth for +// I420 transfer. +const int MAX_INPUT_VIDEO_WIDTH = 200; + // Define flag validators for our flags: static bool ValidatePath(const char* flag_name, const std::string& value) { return !value.empty(); } static bool ValidateDimension(const char* flag_name, WebRtc_Word32 value) { - if (value <= 0) { + if (value <= 0 || value > MAX_INPUT_VIDEO_WIDTH) { return false; } return true; @@ -44,28 +52,115 @@ static const bool dummy1 = google::RegisterFlagValidator(&FLAGS_i420_test_video_path, &ValidatePath); -DEFINE_int32(i420_test_video_width, 0, "The width of the provided video."); +DEFINE_int32(i420_test_video_width, 0, "The width of the provided video." + " This cannot be larger than 200 due to bandwidth concerns"); static const bool dummy2 = google::RegisterFlagValidator(&FLAGS_i420_test_video_width, &ValidateDimension); -DEFINE_int32(i420_test_video_height, 0, "The height of the provided video."); +DEFINE_int32(i420_test_video_height, 0, "The height of the provided video." + " This cannot be larger than 200 due to bandwidth concerns"); static const bool dummy3 = google::RegisterFlagValidator(&FLAGS_i420_test_video_height, &ValidateDimension); class ViEStandardIntegrationTest: public ViEIntegrationTest { + public: + void SetUp() { + std::string test_case_name = + ::testing::UnitTest::GetInstance()->current_test_info()->name(); + + std::string local_preview_filename = test_case_name + "-local-preview.yuv"; + std::string remote_filename = test_case_name + "-remote.yuv"; + + if (!local_file_renderer_.PrepareForRendering(local_preview_filename)) { + FAIL() << "Could not open output file " << local_preview_filename << + " for writing."; + } + if (!remote_file_renderer_.PrepareForRendering(remote_filename)) { + FAIL() << "Could not open output file " << remote_filename << + " for writing."; + } + } + + void TearDown() { + local_file_renderer_.StopRendering(); + remote_file_renderer_.StopRendering(); + + bool test_failed = ::testing::UnitTest::GetInstance()-> + current_test_info()->result()->Failed(); + + if (test_failed) { + // Leave the files for analysis if the test failed + local_file_renderer_.SaveOutputFile("failed-"); + remote_file_renderer_.SaveOutputFile("failed-"); + } else { + // No reason to keep the files if we succeeded + local_file_renderer_.DeleteOutputFile(); + remote_file_renderer_.DeleteOutputFile(); + } + } + protected: + ViEToFileRenderer local_file_renderer_; + ViEToFileRenderer remote_file_renderer_; }; -TEST_F(ViEStandardIntegrationTest, RunsBaseTestWithoutErrors) { +TEST_F(ViEStandardIntegrationTest, RunsBaseStandardTestWithoutErrors) { tests_->ViEAutomatedBaseStandardTest(FLAGS_i420_test_video_path, FLAGS_i420_test_video_width, - FLAGS_i420_test_video_height); + FLAGS_i420_test_video_height, + &local_file_renderer_, + &remote_file_renderer_); + + QualityMetricsResult psnr_result; + int psnr_error = PsnrFromFiles(FLAGS_i420_test_video_path.c_str(), + remote_file_renderer_.output_filename().c_str(), + FLAGS_i420_test_video_width, + FLAGS_i420_test_video_height, + &psnr_result); + ASSERT_EQ(0, psnr_error) << "The PSNR routine failed - output files missing?"; + ASSERT_GT(psnr_result.average, 25); // That is, we want at least 25 dB + + QualityMetricsResult ssim_result; + int ssim_error = SsimFromFiles(FLAGS_i420_test_video_path.c_str(), + remote_file_renderer_.output_filename().c_str(), + FLAGS_i420_test_video_width, + FLAGS_i420_test_video_height, + &ssim_result); + ASSERT_EQ(0, ssim_error) << "The SSIM routine failed - output files missing?"; + ASSERT_GT(ssim_result.average, 0.85f); // 1 = perfect, -1 = terrible } TEST_F(ViEStandardIntegrationTest, RunsCodecTestWithoutErrors) { tests_->ViEAutomatedCodecStandardTest(FLAGS_i420_test_video_path, FLAGS_i420_test_video_width, - FLAGS_i420_test_video_height); + FLAGS_i420_test_video_height, + &local_file_renderer_, + &remote_file_renderer_); + + // We compare the left and right here instead of with the original. The reason + // is that it is hard to say when the three consecutive tests switch over into + // each other, at which point we would have to restart the original to get a + // fair comparison. + QualityMetricsResult psnr_result; + int psnr_error = PsnrFromFiles(local_file_renderer_.output_filename().c_str(), + remote_file_renderer_.output_filename().c_str(), + FLAGS_i420_test_video_width, + FLAGS_i420_test_video_height, + &psnr_result); + ASSERT_EQ(0, psnr_error) << "The PSNR routine failed - output files missing?"; + // This test includes VP8 which is a bit lossy. According to Wikipedia between + // 20-25 is considered OK for transmission codecs and we seem to be getting + // like 21 so 20 seems like a good threshold value here. + EXPECT_GT(psnr_result.average, 20); + + QualityMetricsResult ssim_result; + int ssim_error = SsimFromFiles(local_file_renderer_.output_filename().c_str(), + remote_file_renderer_.output_filename().c_str(), + FLAGS_i420_test_video_width, + FLAGS_i420_test_video_height, + &ssim_result); + ASSERT_EQ(0, ssim_error) << "The SSIM routine failed - output files missing?"; + EXPECT_GT(ssim_result.average, 0.8f); // 1 = perfect, -1 = terrible } // These tests still require a physical camera: @@ -73,7 +168,6 @@ TEST_F(ViEStandardIntegrationTest, RunsCaptureTestWithoutErrors) { ASSERT_EQ(0, tests_->ViECaptureStandardTest()); } - TEST_F(ViEStandardIntegrationTest, RunsEncryptionTestWithoutErrors) { ASSERT_EQ(0, tests_->ViEEncryptionStandardTest()); } diff --git a/src/video_engine/main/test/AutoTest/helpers/vie_fake_camera.cc b/src/video_engine/main/test/AutoTest/helpers/vie_fake_camera.cc index ce701b47c5..387ea8ce21 100644 --- a/src/video_engine/main/test/AutoTest/helpers/vie_fake_camera.cc +++ b/src/video_engine/main/test/AutoTest/helpers/vie_fake_camera.cc @@ -20,8 +20,12 @@ bool StreamVideoFileRepeatedlyIntoCaptureDevice(void* data) { ViEFileCaptureDevice* file_capture_device = reinterpret_cast(data); + // We want to interrupt the camera feeding thread every now and then in order + // to follow the contract for the system_wrappers thread library. 1.5 seconds + // seems about right here. uint64_t time_slice_ms = 1500; uint32_t max_fps = 30; + file_capture_device->ReadFileFor(time_slice_ms, max_fps); return true; diff --git a/src/video_engine/main/test/AutoTest/helpers/vie_file_capture_device.cc b/src/video_engine/main/test/AutoTest/helpers/vie_file_capture_device.cc index 657c7fb2b4..e4caa2a300 100644 --- a/src/video_engine/main/test/AutoTest/helpers/vie_file_capture_device.cc +++ b/src/video_engine/main/test/AutoTest/helpers/vie_file_capture_device.cc @@ -72,8 +72,6 @@ void ViEFileCaptureDevice::ReadFileFor(uint64_t time_slice_ms, webrtc::CriticalSectionScoped cs(*mutex_); assert(input_file_ != NULL); - webrtc::VideoFrame sourceFrame; - sourceFrame.VerifyAndAllocate(frame_length_); unsigned char* frame_buffer = new unsigned char[frame_length_]; webrtc::EventWrapper* sleeper = webrtc::EventWrapper::Create(); diff --git a/src/video_engine/main/test/AutoTest/helpers/vie_to_file_renderer.cc b/src/video_engine/main/test/AutoTest/helpers/vie_to_file_renderer.cc new file mode 100644 index 0000000000..5c2a15c26e --- /dev/null +++ b/src/video_engine/main/test/AutoTest/helpers/vie_to_file_renderer.cc @@ -0,0 +1,89 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "vie_to_file_renderer.h" + +#include + +ViEToFileRenderer::ViEToFileRenderer() + : output_file_(NULL) { +} + +ViEToFileRenderer::~ViEToFileRenderer() { +} + +bool ViEToFileRenderer::PrepareForRendering( + const std::string& output_filename) { + + assert(output_file_ == NULL); + + output_file_ = std::fopen(output_filename.c_str(), "wb"); + if (output_file_ == NULL) { + return false; + } + + output_filename_ = output_filename; + return true; +} + +void ViEToFileRenderer::StopRendering() { + assert(output_file_ != NULL); + std::fclose(output_file_); + output_file_ = NULL; +} + +bool ViEToFileRenderer::SaveOutputFile(const std::string& prefix) { + assert(output_file_ == NULL && output_filename_ != ""); + if (std::rename(output_filename_.c_str(), + (prefix + output_filename_).c_str()) != 0) { + std::perror("Failed to rename output file"); + return false; + } + // Forget about the file + output_filename_ = ""; + return true; +} + +bool ViEToFileRenderer::DeleteOutputFile() { + assert(output_file_ == NULL && output_filename_ != ""); + if (std::remove(output_filename_.c_str()) != 0) { + std::perror("Failed to delete output file"); + return false; + } + output_filename_ = ""; + return true; +} + +const std::string& ViEToFileRenderer::output_filename() const { + assert(output_file_ != NULL); + + return output_filename_; +} + +int ViEToFileRenderer::DeliverFrame(unsigned char *buffer, + int buffer_size, + unsigned int time_stamp) { + assert(output_file_ != NULL); + + int written = std::fwrite(buffer, sizeof(unsigned char), + buffer_size, output_file_); + + if (written == buffer_size) { + return 0; + } else { + return -1; + } +} + +int ViEToFileRenderer::FrameSizeChange(unsigned int width, + unsigned int height, + unsigned int number_of_streams) { + return 0; +} diff --git a/src/video_engine/main/test/AutoTest/helpers/vie_to_file_renderer.h b/src/video_engine/main/test/AutoTest/helpers/vie_to_file_renderer.h new file mode 100644 index 0000000000..8beaf0cfd7 --- /dev/null +++ b/src/video_engine/main/test/AutoTest/helpers/vie_to_file_renderer.h @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2011 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#ifndef SRC_VIDEO_ENGINE_MAIN_TEST_AUTOTEST_HELPERS_VIE_TO_FILE_RENDERER_H_ +#define SRC_VIDEO_ENGINE_MAIN_TEST_AUTOTEST_HELPERS_VIE_TO_FILE_RENDERER_H_ + +#include + +#include + +#include "vie_render.h" + +class ViEToFileRenderer: public webrtc::ExternalRenderer { + public: + ViEToFileRenderer(); + virtual ~ViEToFileRenderer(); + + // Returns false if we fail opening the output filename for writing. + bool PrepareForRendering(const std::string& output_filename); + + // Closes the output file. + void StopRendering(); + + // Deletes the closed output file from the file system. This is one option + // after calling StopRendering, the other being KeepOutputFile. This file + // renderer will forget about the file after this call and can be used again. + bool DeleteOutputFile(); + + // Renames the closed output file to its previous name with the provided + // prefix prepended. This file renderer will forget about the file after this + // call and can be used again. + bool SaveOutputFile(const std::string& prefix); + + // Implementation of ExternalRenderer: + int FrameSizeChange(unsigned int width, unsigned int height, + unsigned int number_of_streams); + + int DeliverFrame(unsigned char* buffer, int buffer_size, + unsigned int time_stamp); + + const std::string& output_filename() const; + + private: + std::FILE* output_file_; + std::string output_filename_; +}; + +#endif // SRC_VIDEO_ENGINE_MAIN_TEST_AUTOTEST_HELPERS_VIE_TO_FILE_RENDERER_H_ diff --git a/src/video_engine/main/test/AutoTest/interface/vie_autotest.h b/src/video_engine/main/test/AutoTest/interface/vie_autotest.h index d87d7b6078..41c1bb2b72 100644 --- a/src/video_engine/main/test/AutoTest/interface/vie_autotest.h +++ b/src/video_engine/main/test/AutoTest/interface/vie_autotest.h @@ -40,6 +40,7 @@ #endif class tbInterfaces; +class ViEToFileRenderer; class ViEAutoTest { @@ -63,9 +64,14 @@ public: int ViEBaseAPITest(); // This is a variant of the base standard test, meant to run in GTest. - void ViEAutomatedBaseStandardTest(const std::string& pathToTestI420Video, + // The first three arguments describes the file to use as fake camera + // input. The file renderer arguments describe where to put the output + // from the left and right windows, respectively. + void ViEAutomatedBaseStandardTest(const std::string& i420_test_video_path, int width, - int height); + int height, + ViEToFileRenderer* local_file_renderer, + ViEToFileRenderer* remote_file_renderer); // vie_autotest_capture.cc int ViECaptureStandardTest(); @@ -81,7 +87,9 @@ public: void ViEAutomatedCodecStandardTest(const std::string& pathToTestI420Video, int width, - int height); + int height, + ViEToFileRenderer* local_file_renderer, + ViEToFileRenderer* remote_file_renderer); // vie_autotest_encryption.cc int ViEEncryptionStandardTest(); @@ -125,16 +133,25 @@ private: int* number_of_errors, webrtc::VideoCaptureModule** device_video); - webrtc::ViERender *RenderInBothWindows(webrtc::VideoEngine * ptrViE, - int & numberOfErrors, int captureId, - int videoChannel); + void RenderInWindow(webrtc::ViERender* video_render_interface, + int* numberOfErrors, + int frame_provider_id, + void* os_window, + float z_index); + void RenderToFile(webrtc::ViERender* renderer_interface, + int render_id, + ViEToFileRenderer *to_file_renderer); void PrintAudioCodec(const webrtc::CodecInst audioCodec); void PrintVideoCodec(const webrtc::VideoCodec videoCodec); void RunCodecTestInternal(const tbInterfaces& interfaces, int & numberOfErrors, - int captureId); + int captureId, + int forced_codec_width, + int forced_codec_height, + ViEToFileRenderer* left_file_renderer, + ViEToFileRenderer* right_file_renderer); void* _window1; void* _window2; diff --git a/src/video_engine/main/test/AutoTest/source/vie_autotest_base.cc b/src/video_engine/main/test/AutoTest/source/vie_autotest_base.cc index 27ab17d970..0afa8f2c72 100644 --- a/src/video_engine/main/test/AutoTest/source/vie_autotest_base.cc +++ b/src/video_engine/main/test/AutoTest/source/vie_autotest_base.cc @@ -8,17 +8,13 @@ * be found in the AUTHORS file in the root of the source tree. */ -// -// vie_autotest_base.cc -// - +#include "engine_configurations.h" #include "gtest/gtest.h" #include "video_capture_factory.h" #include "vie_autotest_defines.h" #include "vie_autotest.h" #include "vie_fake_camera.h" -#include "vie_file_capture_device.h" -#include "thread_wrapper.h" +#include "vie_to_file_renderer.h" class BaseObserver : public webrtc::ViEBaseObserver { public: @@ -110,36 +106,24 @@ webrtc::ViERTP_RTCP *ConfigureRtpRtcp(webrtc::VideoEngine * ptrViE, return ptrViERtpRtcp; } -webrtc::ViERender *ViEAutoTest::RenderInBothWindows( - webrtc::VideoEngine * ptrViE, int & numberOfErrors, - int captureId, int videoChannel) { - webrtc::ViERender *ptrViERender = webrtc::ViERender::GetInterface(ptrViE); - numberOfErrors += ViETest::TestError(ptrViERender != NULL, - "ERROR: %s at line %d", __FUNCTION__, - __LINE__); - int error = ptrViERender->RegisterVideoRenderModule(*_vrm1); +void ViEAutoTest::RenderInWindow(webrtc::ViERender* video_render_interface, + int* numberOfErrors, + int frame_provider_id, + void* os_window, + float z_index) { + int error = video_render_interface->AddRenderer(frame_provider_id, os_window, + z_index, 0.0, 0.0, 1.0, 1.0); + *numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + + error = video_render_interface->StartRender(frame_provider_id); numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", __FUNCTION__, __LINE__); - error = ptrViERender->AddRenderer(captureId, _window1, 0, 0.0, - 0.0, 1.0, 1.0); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViERender->StartRender(captureId); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViERender->RegisterVideoRenderModule(*_vrm2); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViERender->AddRenderer(videoChannel, _window2, 1, 0.0, - 0.0, 1.0, 1.0); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViERender->StartRender(videoChannel); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - return ptrViERender; } +// Tests a I420-to-I420 call. This test exercises the most basic WebRTC ViE +// functionality by setting up the codec interface to recognize the most common +// codecs, and the initiating a I420 call. webrtc::ViENetwork *TestCallSetup(webrtc::ViECodec * ptrViECodec, int & numberOfErrors, int videoChannel, @@ -203,6 +187,14 @@ webrtc::ViENetwork *TestCallSetup(webrtc::ViECodec * ptrViECodec, ViETest::Log("Call started"); ViETest::Log("You should see a local preview from camera %s" " in window 1 and the remote video in window 2.", deviceName); + + AutoTestSleep(KAutoTestSleepTimeMs); + + // Done + error = ptrViEBase->StopSend(videoChannel); + numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + return ptrViENetwork; } @@ -224,9 +216,6 @@ void StopEverything(webrtc::ViEBase * ptrViEBase, numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", __FUNCTION__, __LINE__); error = ptrViERender->DeRegisterVideoRenderModule(*vrm2); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViEBase->StopSend(videoChannel); numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", __FUNCTION__, __LINE__); error = ptrViERender->RemoveRenderer(captureId); @@ -323,8 +312,21 @@ int ViEAutoTest::ViEBaseStandardTest() { webrtc::ViERTP_RTCP *ptrViERtpRtcp = ConfigureRtpRtcp(ptrViE, numberOfErrors, videoChannel); - webrtc::ViERender *ptrViERender = - RenderInBothWindows(ptrViE, numberOfErrors, captureId, videoChannel); + + webrtc::ViERender *ptrViERender = webrtc::ViERender::GetInterface(ptrViE); + ViETest::TestError(ptrViERender != NULL, + "ERROR: %s at line %d", __FUNCTION__, + __LINE__); + + error = ptrViERender->RegisterVideoRenderModule(*_vrm1); + numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + error = ptrViERender->RegisterVideoRenderModule(*_vrm2); + numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + + RenderInWindow(ptrViERender, &numberOfErrors, captureId, _window1, 0); + RenderInWindow(ptrViERender, &numberOfErrors, videoChannel, _window2, 1); webrtc::ViECodec *ptrViECodec = webrtc::ViECodec::GetInterface(ptrViE); numberOfErrors += ViETest::TestError(ptrViECodec != NULL, @@ -332,14 +334,12 @@ int ViEAutoTest::ViEBaseStandardTest() { __LINE__); // *************************************************************** - // Set up the call and wait. + // Run the actual test: // *************************************************************** webrtc::ViENetwork *ptrViENetwork = - TestCallSetup(ptrViECodec, numberOfErrors, videoChannel, - ptrViE, ptrViEBase, deviceName); - - AutoTestSleep(KAutoTestSleepTimeMs); + TestCallSetup(ptrViECodec, numberOfErrors, videoChannel, + ptrViE, ptrViEBase, deviceName); // *************************************************************** // Testing finished. Tear down Video Engine @@ -407,10 +407,10 @@ int ViEAutoTest::ViEBaseExtendedTest() { // "ERROR: %s at line %d", // __FUNCTION__, __LINE__); - ViETest::Log(" "); - ViETest::Log(" ViEBase Extended Test PASSED!"); - ViETest::Log("========================================"); - ViETest::Log(" "); + ViETest::Log(" "); + ViETest::Log(" ViEBase Extended Test PASSED!"); + ViETest::Log("========================================"); + ViETest::Log(" "); return 0; } @@ -591,71 +591,6 @@ int ViEAutoTest::ViEBaseAPITest() { return 0; } -void ViEAutoTest::ViEAutomatedBaseStandardTest( - const std::string& pathToTestI420Video, int width, int height) { - int ignoredNumberOfErrors; - - // Initialize the test: - webrtc::VideoEngine* ptrViE = - InitializeVideoEngine(ignoredNumberOfErrors); - webrtc::ViEBase *ptrViEBase = - InitializeViEBase(ptrViE, ignoredNumberOfErrors); - - int videoChannel = -1; - webrtc::ViECapture *ptrViECapture = - InitializeChannel(ptrViEBase, videoChannel, - ignoredNumberOfErrors, ptrViE); - - ViEFakeCamera fakeCamera(ptrViECapture); - if (!fakeCamera.StartCameraInNewThread(pathToTestI420Video, width, height)) { - // No point in continuing if we have no proper video source - ViETest::TestError(false, "ERROR: %s at line %d: " - "Could not open input video %s: aborting test...", - __FUNCTION__, __LINE__, pathToTestI420Video.c_str()); - return; - } - int captureId = fakeCamera.capture_id(); - - // Apparently, we need to connect external capture devices, but we should - // not start them since the external device is not a proper device. - ConnectCaptureDevice(ptrViECapture, captureId, videoChannel, - ignoredNumberOfErrors); - - webrtc::ViERTP_RTCP *ptrViERtpRtcp = - ConfigureRtpRtcp(ptrViE, ignoredNumberOfErrors, videoChannel); - webrtc::ViERender *ptrViERender = - RenderInBothWindows(ptrViE, ignoredNumberOfErrors, - captureId, videoChannel); - - webrtc::ViECodec *ptrViECodec = webrtc::ViECodec::GetInterface(ptrViE); - ignoredNumberOfErrors += - ViETest::TestError(ptrViECodec != NULL, - "ERROR: %s at line %d", __FUNCTION__, - __LINE__); - - // Run the test itself: - const WebRtc_UWord8* deviceName = - reinterpret_cast("Fake Capture Device"); - webrtc::ViENetwork *ptrViENetwork = - TestCallSetup(ptrViECodec, ignoredNumberOfErrors, videoChannel, - ptrViE, ptrViEBase, deviceName); - - AutoTestSleep(KAutoTestSleepTimeMs); - - StopEverything(ptrViEBase, videoChannel, ignoredNumberOfErrors, ptrViERender, - captureId, ptrViECapture, _vrm1, _vrm2); - - // Stop sending data, clean up the camera thread and release the capture - // device. Note that this all happens after StopEverything, so this is - // tests that the system doesn't mind that the external capture device sends - // data after rendering has been stopped. - fakeCamera.StopCamera(); - - ReleaseEverything(ptrViECapture, ignoredNumberOfErrors, ptrViEBase, - videoChannel, ptrViECodec, ptrViERtpRtcp, ptrViERender, - ptrViENetwork, ptrViE); -} - void ViEAutoTest::FindCaptureDeviceOnSystem( webrtc::ViECapture* capture, unsigned char* device_name, @@ -700,3 +635,95 @@ void ViEAutoTest::FindCaptureDeviceOnSystem( capture_device_set, "ERROR: %s at line %d - Could not set capture device", __FUNCTION__, __LINE__); } + +void ViEAutoTest::RenderToFile(webrtc::ViERender* renderer_interface, + int render_id, + ViEToFileRenderer *to_file_renderer) +{ + int result = renderer_interface->AddRenderer(render_id, + webrtc::kVideoI420, + to_file_renderer); + ViETest::TestError(result == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + result = renderer_interface->StartRender(render_id); + ViETest::TestError(result == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); +} + +void ViEAutoTest::ViEAutomatedBaseStandardTest( + const std::string& i420_test_video_path, + int width, + int height, + ViEToFileRenderer* local_file_renderer, + ViEToFileRenderer* remote_file_renderer) { + int ignored; + + // Initialize the test: + webrtc::VideoEngine* video_engine = + InitializeVideoEngine(ignored); + webrtc::ViEBase *base_interface = + InitializeViEBase(video_engine, ignored); + + int video_channel = -1; + webrtc::ViECapture *capture_interface = + InitializeChannel(base_interface, video_channel, ignored, video_engine); + + ViEFakeCamera fake_camera(capture_interface); + if (!fake_camera.StartCameraInNewThread(i420_test_video_path, + width, + height)) { + // No point in continuing if we have no proper video source + ViETest::TestError(false, "ERROR: %s at line %d: " + "Could not open input video %s: aborting test...", + __FUNCTION__, __LINE__, i420_test_video_path.c_str()); + return; + } + int capture_id = fake_camera.capture_id(); + + // Apparently, we need to connect external capture devices, but we should + // not start them since the external device is not a proper device. + ConnectCaptureDevice(capture_interface, capture_id, video_channel, + ignored); + + webrtc::ViERTP_RTCP *rtcp_interface = + ConfigureRtpRtcp(video_engine, ignored, video_channel); + + webrtc::ViERender *render_interface = + webrtc::ViERender::GetInterface(video_engine); + ViETest::TestError(render_interface != NULL, + "ERROR: %s at line %d", __FUNCTION__, + __LINE__); + + render_interface->RegisterVideoRenderModule(*_vrm1); + render_interface->RegisterVideoRenderModule(*_vrm2); + RenderToFile(render_interface, capture_id, local_file_renderer); + RenderToFile(render_interface, video_channel, remote_file_renderer); + + webrtc::ViECodec *codec_interface = + webrtc::ViECodec::GetInterface(video_engine); + ViETest::TestError(codec_interface != NULL, + "ERROR: %s at line %d", __FUNCTION__, + __LINE__); + + // Run the test itself: + const WebRtc_UWord8* device_name = + reinterpret_cast("Fake Capture Device"); + webrtc::ViENetwork *network_interface = + TestCallSetup(codec_interface, ignored, video_channel, + video_engine, base_interface, device_name); + + AutoTestSleep(KAutoTestSleepTimeMs); + + StopEverything(base_interface, video_channel, ignored, render_interface, + capture_id, capture_interface, _vrm1, _vrm2); + + // Stop sending data, clean up the camera thread and release the capture + // device. Note that this all happens after StopEverything, so this + // tests that the system doesn't mind that the external capture device sends + // data after rendering has been stopped. + fake_camera.StopCamera(); + + ReleaseEverything(capture_interface, ignored, base_interface, + video_channel, codec_interface, rtcp_interface, + render_interface, network_interface, video_engine); +} diff --git a/src/video_engine/main/test/AutoTest/source/vie_autotest_codec.cc b/src/video_engine/main/test/AutoTest/source/vie_autotest_codec.cc index ca60073019..96ea97b81b 100644 --- a/src/video_engine/main/test/AutoTest/source/vie_autotest_codec.cc +++ b/src/video_engine/main/test/AutoTest/source/vie_autotest_codec.cc @@ -30,8 +30,10 @@ #include "vie_rtp_rtcp.h" #include "voe_base.h" -class ViEAutotestCodecObserever: public webrtc::ViEEncoderObserver, - public webrtc::ViEDecoderObserver +const int kDoNotForceResolution = 0; + +class ViEAutotestCodecObserver: public webrtc::ViEEncoderObserver, + public webrtc::ViEDecoderObserver { public: int incomingCodecCalled; @@ -49,7 +51,7 @@ public: webrtc::VideoCodec incomingCodec; - ViEAutotestCodecObserever() + ViEAutotestCodecObserver() { incomingCodecCalled = 0; incomingRatecalled = 0; @@ -118,11 +120,188 @@ public: } }; +// Helper functions + +// Finds a codec in the codec list. Returns 0 on success, nonzero otherwise. +// The resulting codec is filled into result on success but is zeroed out +// on failure. +int FindSpecificCodec(webrtc::VideoCodecType of_type, + webrtc::ViECodec* codec_interface, + webrtc::VideoCodec* result) { + + memset(result, 1, sizeof(webrtc::VideoCodec)); + + for (int i = 0; i < codec_interface->NumberOfCodecs(); i++) { + webrtc::VideoCodec codec; + if (codec_interface->GetCodec(i, codec) != 0) { + return -1; + } + if (codec.codecType == of_type) { + // Done + *result = codec; + return 0; + } + } + // Didn't find it + return -1; +} + +// Sets the video codec's resolution info to something suitable based on each +// codec's quirks, except if the forced* variables are != kDoNotForceResolution. +void SetSuitableResolution(webrtc::VideoCodec* video_codec, + int forced_codec_width, + int forced_codec_height) { + if (forced_codec_width != kDoNotForceResolution && + forced_codec_height != kDoNotForceResolution) { + video_codec->width = forced_codec_width; + video_codec->height = forced_codec_height; + } else if (video_codec->codecType == webrtc::kVideoCodecI420) { + // I420 is very bandwidth heavy, so limit it here + video_codec->width = 176; + video_codec->height = 144; + } else if (video_codec->codecType != webrtc::kVideoCodecH263) { + // Otherwise go with 640x480, except for H263 which can do whatever + // it pleases. + video_codec->width = 640; + video_codec->height = 480; + } +} + +// Tests that a codec actually renders frames by registering a basic +// render effect filter on the codec and then running it. This test is +// quite lenient on the number of frames that get rendered, so it should not +// be seen as a end-user-visible quality measure - it is more a sanity check +// that the codec at least gets some frames through. +void TestCodecImageProcess(webrtc::VideoCodec video_codec, + webrtc::ViECodec* codec_interface, + int video_channel, + int* number_of_errors, + webrtc::ViEImageProcess* image_process) { + + int error = codec_interface->SetSendCodec(video_channel, video_codec); + *number_of_errors += ViETest::TestError(error == 0, + "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + ViEAutoTestEffectFilter frame_counter; + error = image_process->RegisterRenderEffectFilter(video_channel, + frame_counter); + *number_of_errors += ViETest::TestError(error == 0, + "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + AutoTestSleep (KAutoTestSleepTimeMs); + + int max_number_of_rendered_frames = video_codec.maxFramerate * + KAutoTestSleepTimeMs / 1000; + + if (video_codec.codecType == webrtc::kVideoCodecI420) { + // Due to that I420 needs a huge bandwidth, rate control can set + // frame rate very low. This happen since we use the same channel + // as we just tested with vp8. + *number_of_errors += ViETest::TestError(frame_counter.numFrames > 0, + "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + } else { +#ifdef WEBRTC_ANDROID + // Special case to get the autotest to pass on some slow devices + *number_of_errors += + ViETest::TestError(frameCounter.numFrames + > max_number_of_rendered_frames / 6, + "ERROR: %s at line %d", + __FUNCTION__, __LINE__); +#else + *number_of_errors += ViETest::TestError(frame_counter.numFrames + > max_number_of_rendered_frames / 4, + "ERROR: %s at line %d", + __FUNCTION__, __LINE__); +#endif + } + error = image_process->DeregisterRenderEffectFilter(video_channel); + *number_of_errors += ViETest::TestError(error == 0, + "ERROR: %s at line %d", + __FUNCTION__, __LINE__); +} + +void SetSendCodec(webrtc::VideoCodecType of_type, + webrtc::ViECodec* codec_interface, + int video_channel, + int* number_of_errors, + int forced_codec_width, + int forced_codec_height) { + webrtc::VideoCodec codec; + int error = FindSpecificCodec(of_type, codec_interface, &codec); + *number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + + SetSuitableResolution(&codec, forced_codec_width, forced_codec_height); + + error = codec_interface->SetSendCodec(video_channel, codec); + *number_of_errors += ViETest::TestError(error == 0, + "ERROR: %s at line %d", + __FUNCTION__, __LINE__); +} + +// Test switching from i420 to VP8 as send codec and make sure that +// the codec observer gets called after the switch. +void TestCodecCallbacks(webrtc::ViEBase *& base_interface, + webrtc::ViECodec *codec_interface, + int video_channel, + int* number_of_errors, + int forced_codec_width, + int forced_codec_height) { + + // Set I420 as send codec so we don't make any assumptions about what + // we currently have as send codec: + SetSendCodec(webrtc::kVideoCodecI420, codec_interface, video_channel, + number_of_errors, forced_codec_width, forced_codec_height); + + // Register the observer: + ViEAutotestCodecObserver codec_observer; + int error = codec_interface->RegisterEncoderObserver(video_channel, + codec_observer); + *number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + error = codec_interface->RegisterDecoderObserver(video_channel, + codec_observer); + *number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + + // Make the switch + ViETest::Log("Testing codec callbacks..."); + + SetSendCodec(webrtc::kVideoCodecVP8, codec_interface, video_channel, + number_of_errors, forced_codec_width, forced_codec_height); + + AutoTestSleep (KAutoTestSleepTimeMs); + + // Verify that we got the right codec + *number_of_errors += ViETest::TestError( + codec_observer.incomingCodec.codecType == webrtc::kVideoCodecVP8, + "ERROR: %s at line %d", __FUNCTION__, __LINE__); + + // Clean up + error = codec_interface->DeregisterEncoderObserver(video_channel); + *number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + error = codec_interface->DeregisterDecoderObserver(video_channel); + *number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + *number_of_errors += ViETest::TestError( + codec_observer.incomingCodecCalled > 0, + "ERROR: %s at line %d", __FUNCTION__, __LINE__); + *number_of_errors += ViETest::TestError( + codec_observer.incomingRatecalled > 0, + "ERROR: %s at line %d", __FUNCTION__, __LINE__); + *number_of_errors += ViETest::TestError( + codec_observer.outgoingRatecalled > 0, + "ERROR: %s at line %d", __FUNCTION__, __LINE__); +} + void ViEAutoTest::ViEAutomatedCodecStandardTest( const std::string& i420_video_file, int width, - int height) { - + int height, + ViEToFileRenderer* local_file_renderer, + ViEToFileRenderer* remote_file_renderer) { int ignored = 0; tbInterfaces interfaces = tbInterfaces("ViECodecAutomatedStandardTest", @@ -137,7 +316,11 @@ void ViEAutoTest::ViEAutomatedCodecStandardTest( return; } - RunCodecTestInternal(interfaces, ignored, fake_camera.capture_id()); + // Force the codec resolution to what our input video is so we can make + // comparisons later. Our comparison algorithms wouldn't like scaling. + RunCodecTestInternal(interfaces, ignored, fake_camera.capture_id(), + width, height, local_file_renderer, + remote_file_renderer); fake_camera.StopCamera(); } @@ -155,9 +338,9 @@ int ViEAutoTest::ViECodecStandardTest() tbCaptureDevice capture_device = tbCaptureDevice(interfaces, number_of_errors); - RunCodecTestInternal(interfaces, - number_of_errors, - capture_device.captureId); + RunCodecTestInternal(interfaces, number_of_errors, capture_device.captureId, + kDoNotForceResolution, kDoNotForceResolution, + NULL, NULL); if (number_of_errors > 0) { @@ -294,7 +477,7 @@ int ViEAutoTest::ViECodecExtendedTest() numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", __FUNCTION__, __LINE__); - ViEAutotestCodecObserever codecObserver; + ViEAutotestCodecObserver codecObserver; error = ptrViECodec->RegisterEncoderObserver(videoChannel, codecObserver); numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", @@ -377,7 +560,7 @@ int ViEAutoTest::ViECodecExtendedTest() numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", __FUNCTION__, __LINE__); - ViEAutotestCodecObserever codecObserver1; + ViEAutotestCodecObserver codecObserver1; error = ViE.ptrViECodec->RegisterEncoderObserver(videoChannel1, codecObserver1); numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", @@ -454,7 +637,7 @@ int ViEAutoTest::ViECodecExtendedTest() __FUNCTION__, __LINE__); } - ViEAutotestCodecObserever codecObserver2; + ViEAutotestCodecObserver codecObserver2; error = ViE.ptrViECodec->RegisterDecoderObserver(videoChannel2, codecObserver2); numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", @@ -539,7 +722,7 @@ int ViEAutoTest::ViECodecExtendedTest() __FUNCTION__, __LINE__); } - ViEAutotestCodecObserever codecObserver3; + ViEAutotestCodecObserver codecObserver3; error = ViE.ptrViECodec->RegisterDecoderObserver(videoChannel3, codecObserver3); numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", @@ -609,7 +792,6 @@ int ViEAutoTest::ViECodecExtendedTest() ViETest::Log("========================================"); ViETest::Log(" "); return 0; - } int ViEAutoTest::ViECodecAPITest() @@ -653,7 +835,6 @@ int ViEAutoTest::ViECodecAPITest() // Engine ready. Begin testing class //*************************************************************** - // // SendCodec // @@ -665,38 +846,15 @@ int ViEAutoTest::ViECodecAPITest() "ERROR: %s at line %d", __FUNCTION__, __LINE__); - for (int idx = 0; idx < numberOfCodecs; idx++) - { - error = ptrViECodec->GetCodec(idx, videoCodec); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - if (videoCodec.codecType == webrtc::kVideoCodecVP8) - { - error = ptrViECodec->SetSendCodec(videoChannel, videoCodec); - numberOfErrors += ViETest::TestError(error == 0, - "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - break; - } - } + SetSendCodec(webrtc::kVideoCodecVP8, ptrViECodec, videoChannel, + &numberOfErrors, kDoNotForceResolution, kDoNotForceResolution); + memset(&videoCodec, 0, sizeof(videoCodec)); error = ptrViECodec->GetSendCodec(videoChannel, videoCodec); assert(videoCodec.codecType == webrtc::kVideoCodecVP8); - for (int idx = 0; idx < numberOfCodecs; idx++) - { - error = ptrViECodec->GetCodec(idx, videoCodec); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - if (videoCodec.codecType == webrtc::kVideoCodecI420) - { - error = ptrViECodec->SetSendCodec(videoChannel, videoCodec); - numberOfErrors += ViETest::TestError(error == 0, - "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - break; - } - } + SetSendCodec(webrtc::kVideoCodecI420, ptrViECodec, videoChannel, + &numberOfErrors, kDoNotForceResolution, kDoNotForceResolution); memset(&videoCodec, 0, sizeof(videoCodec)); error = ptrViECodec->GetSendCodec(videoChannel, videoCodec); assert(videoCodec.codecType == webrtc::kVideoCodecI420); @@ -1023,269 +1181,150 @@ int ViEAutoTest::ViECodecExternalCodecTest() #endif } -void ViEAutoTest::RunCodecTestInternal(const tbInterfaces& interfaces, - int & numberOfErrors, - int captureId) -{ - webrtc::VideoEngine *ptrViE = interfaces.ptrViE; - webrtc::ViEBase *ptrViEBase = interfaces.ptrViEBase; - webrtc::ViECapture *ptrViECapture = interfaces.ptrViECapture; - webrtc::ViERender *ptrViERender = interfaces.ptrViERender; - webrtc::ViECodec *ptrViECodec = interfaces.ptrViECodec; - webrtc::ViERTP_RTCP *ptrViERtpRtcp = interfaces.ptrViERtpRtcp; - webrtc::ViENetwork *ptrViENetwork = interfaces.ptrViENetwork; - int videoChannel = -1; +void ViEAutoTest::RunCodecTestInternal( + const tbInterfaces& interfaces, + int & number_of_errors, + int capture_id, + int forced_codec_width, + int forced_codec_height, + ViEToFileRenderer* local_file_renderer, + ViEToFileRenderer* remote_file_renderer) { - int error = ptrViEBase->CreateChannel(videoChannel); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViECapture->ConnectCaptureDevice(captureId, videoChannel); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViERtpRtcp->SetRTCPStatus(videoChannel, - webrtc::kRtcpCompound_RFC4585); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViERtpRtcp-> - SetKeyFrameRequestMethod(videoChannel, + webrtc::VideoEngine *video_engine_interface = interfaces.ptrViE; + webrtc::ViEBase *base_interface = interfaces.ptrViEBase; + webrtc::ViECapture *capture_interface = interfaces.ptrViECapture; + webrtc::ViERender *render_interface = interfaces.ptrViERender; + webrtc::ViECodec *codec_interface = interfaces.ptrViECodec; + webrtc::ViERTP_RTCP *rtcp_interface = interfaces.ptrViERtpRtcp; + webrtc::ViENetwork *network_interface = interfaces.ptrViENetwork; + int video_channel = -1; + + int error = base_interface->CreateChannel(video_channel); + number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + error = capture_interface->ConnectCaptureDevice(capture_id, video_channel); + number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + error = rtcp_interface->SetRTCPStatus(video_channel, + webrtc::kRtcpCompound_RFC4585); + number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + error = rtcp_interface-> + SetKeyFrameRequestMethod(video_channel, webrtc::kViEKeyFrameRequestPliRtcp); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViERtpRtcp->SetTMMBRStatus(videoChannel, true); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViERender->AddRenderer(captureId, _window1, 0, 0.0, 0.0, 1.0, 1.0); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViERender->AddRenderer(videoChannel, _window2, 1, 0.0, 0.0, - 1.0, 1.0); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViERender->StartRender(captureId); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViERender->StartRender(videoChannel); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - - //*************************************************************** - // Engine ready. Begin testing class - //*************************************************************** - webrtc::VideoCodec videoCodec; - memset(&videoCodec, 0, sizeof (webrtc::VideoCodec)); - for (int idx = 0; idx < ptrViECodec->NumberOfCodecs(); idx++) - { - error = ptrViECodec->GetCodec(idx, videoCodec); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + error = rtcp_interface->SetTMMBRStatus(video_channel, true); + number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", __FUNCTION__, __LINE__); - if (videoCodec.codecType != webrtc::kVideoCodecH263 - && videoCodec.codecType != webrtc::kVideoCodecI420) - { - videoCodec.width = 640; - videoCodec.height = 480; - } - if(videoCodec.codecType == webrtc::kVideoCodecI420) - { - videoCodec.width=176; - videoCodec.height=144; - } - error = ptrViECodec->SetReceiveCodec(videoChannel, videoCodec); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); + if (local_file_renderer && remote_file_renderer) { + RenderToFile(render_interface, capture_id, local_file_renderer); + RenderToFile(render_interface, video_channel, remote_file_renderer); + } else { + RenderInWindow(render_interface, &number_of_errors, + capture_id, _window1, 0); + RenderInWindow(render_interface, &number_of_errors, + video_channel, _window2, 1); } - for (int idx = 0; idx < ptrViECodec->NumberOfCodecs(); idx++) - { - error = ptrViECodec->GetCodec(idx, videoCodec); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - if (videoCodec.codecType == webrtc::kVideoCodecVP8) - { - error = ptrViECodec->SetSendCodec(videoChannel, videoCodec); - numberOfErrors += ViETest::TestError(error == 0, - "ERROR: %s at line %d", + + // *************************************************************** + // Engine ready. Begin testing class + // *************************************************************** + webrtc::VideoCodec video_codec; + webrtc::VideoCodec vp8_codec; + memset(&video_codec, 0, sizeof (webrtc::VideoCodec)); + memset(&vp8_codec, 0, sizeof (webrtc::VideoCodec)); + + // Set up all receive codecs. This sets up a mapping in the codec interface + // which makes it able to recognize all receive codecs based on payload type. + for (int idx = 0; idx < codec_interface->NumberOfCodecs(); idx++) { + error = codec_interface->GetCodec(idx, video_codec); + number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + SetSuitableResolution(&video_codec, + forced_codec_width, + forced_codec_height); + + error = codec_interface->SetReceiveCodec(video_channel, video_codec); + number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", __FUNCTION__, __LINE__); - break; - } } - const char *ipAddress = "127.0.0.1"; - const unsigned short rtpPort = 6000; - error = ptrViENetwork->SetLocalReceiver(videoChannel, rtpPort); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViEBase->StartReceive(videoChannel); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViENetwork->SetSendDestination(videoChannel, ipAddress, rtpPort); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViEBase->StartSend(videoChannel); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - // - // Make sure all codecs runs - // - { - webrtc::ViEImageProcess *ptrViEImageProcess = - webrtc::ViEImageProcess::GetInterface(ptrViE); - ViEAutotestCodecObserever codecObserver; - error = ptrViECodec->RegisterDecoderObserver(videoChannel, codecObserver); - numberOfErrors += ViETest::TestError(error == 0, + const char *ip_address = "127.0.0.1"; + const unsigned short rtp_port = 6000; + error = network_interface->SetLocalReceiver(video_channel, rtp_port); + number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + error = base_interface->StartReceive(video_channel); + number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + error = network_interface->SetSendDestination(video_channel, ip_address, + rtp_port); + number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + error = base_interface->StartSend(video_channel); + number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + // Run all found codecs + webrtc::ViEImageProcess *image_process = + webrtc::ViEImageProcess::GetInterface(video_engine_interface); + number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", __FUNCTION__, __LINE__); - ViETest::Log("Loop through all codecs for %d seconds", - KAutoTestSleepTimeMs / 1000); - for (int idx = 0; idx < ptrViECodec->NumberOfCodecs() - 2; idx++) - { - error = ptrViECodec->GetCodec(idx, videoCodec); - numberOfErrors += ViETest::TestError(error == 0, + + ViETest::Log("Loop through all codecs for %d seconds", + KAutoTestSleepTimeMs / 1000); + for (int i = 0; i < codec_interface->NumberOfCodecs(); i++) { + error = codec_interface->GetCodec(i, video_codec); + number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", __FUNCTION__, __LINE__); - if (videoCodec.codecType != webrtc::kVideoCodecMPEG4) - { - if(videoCodec.codecType == webrtc::kVideoCodecI420) - { - // Lower resolution to sockkets keep up. - videoCodec.width=176; - videoCodec.height=144; - videoCodec.maxFramerate=15; - } - error = ptrViECodec->SetSendCodec(videoChannel, videoCodec); - numberOfErrors += ViETest::TestError(error == 0, - "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - ViETest::Log("\t %d. %s", idx, videoCodec.plName); - ViEAutoTestEffectFilter frameCounter; - error = ptrViEImageProcess->RegisterRenderEffectFilter( - videoChannel, - frameCounter); - numberOfErrors += ViETest::TestError(error == 0, - "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - - AutoTestSleep( KAutoTestSleepTimeMs); - - // Verify we've received and decoded correct payload - numberOfErrors += ViETest::TestError( - codecObserver.incomingCodec.codecType - == videoCodec.codecType, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - - int maxNumberOfRenderedFrames = videoCodec.maxFramerate - * KAutoTestSleepTimeMs / 1000; - - if(videoCodec.codecType == webrtc::kVideoCodecI420) - { - // Due to that I420 needs a huge bandwidth, rate control can set - // frame rate very low. This happen since we use the same channel - // as we just tested with vp8. - numberOfErrors += ViETest::TestError(frameCounter.numFrames>0, - "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - } - else - { - -#ifdef WEBRTC_ANDROID - // Special case to get the autotest to pass on some slow devices - numberOfErrors += - ViETest::TestError(frameCounter.numFrames - > maxNumberOfRenderedFrames / 6, - "ERROR: %s at line %d", - __FUNCTION__, __LINE__); -#else - numberOfErrors += ViETest::TestError(frameCounter.numFrames - > maxNumberOfRenderedFrames / 4, - "ERROR: %s at line %d", - __FUNCTION__, __LINE__); -#endif - } - - error = ptrViEImageProcess->DeregisterRenderEffectFilter( - videoChannel); - numberOfErrors += ViETest::TestError(error == 0, - "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - } - else - { - ViETest::Log("\t %d. %s not tested", idx, videoCodec.plName); - } + if (video_codec.codecType == webrtc::kVideoCodecMPEG4 || + video_codec.codecType == webrtc::kVideoCodecRED || + video_codec.codecType == webrtc::kVideoCodecULPFEC) { + ViETest::Log("\t %d. %s not tested", i, video_codec.plName); + } else { + ViETest::Log("\t %d. %s", i, video_codec.plName); + SetSuitableResolution(&video_codec, + forced_codec_width, forced_codec_height); + TestCodecImageProcess(video_codec, codec_interface, video_channel, + &number_of_errors, image_process); } - ptrViEImageProcess->Release(); - error = ptrViECodec->DeregisterDecoderObserver(videoChannel); - numberOfErrors += ViETest::TestError(error == 0, - "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - ViETest::Log("Done!"); } - // - // Callbacks - // - ViEAutotestCodecObserever codecObserver; - error = ptrViECodec->RegisterEncoderObserver(videoChannel, codecObserver); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViECodec->RegisterDecoderObserver(videoChannel, codecObserver); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - ViETest::Log("\nTesting codec callbacks..."); - for (int idx = 0; idx < ptrViECodec->NumberOfCodecs(); idx++) - { - error = ptrViECodec->GetCodec(idx, videoCodec); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + image_process->Release(); + + TestCodecCallbacks(base_interface, codec_interface, video_channel, + &number_of_errors, forced_codec_width, + forced_codec_height); + + ViETest::Log("Done!"); + + // *************************************************************** + // Testing finished. Tear down Video Engine + // *************************************************************** + error = base_interface->StopSend(video_channel); + number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", __FUNCTION__, __LINE__); - if (videoCodec.codecType == webrtc::kVideoCodecVP8) - { - error = ptrViECodec->SetSendCodec(videoChannel, videoCodec); - numberOfErrors += ViETest::TestError(error == 0, - "ERROR: %s at line %d", + error = base_interface->StopReceive(video_channel); + number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + error = render_interface->StopRender(capture_id); + number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", __FUNCTION__, __LINE__); - break; - } - } - AutoTestSleep (KAutoTestSleepTimeMs); - error = ptrViEBase->StopSend(videoChannel); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViECodec->DeregisterEncoderObserver(videoChannel); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViECodec->DeregisterDecoderObserver(videoChannel); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - numberOfErrors += ViETest::TestError(codecObserver.incomingCodecCalled > 0, - "ERROR: %s at line %d", __FUNCTION__, - __LINE__); - numberOfErrors += ViETest::TestError(codecObserver.incomingRatecalled > 0, - "ERROR: %s at line %d", __FUNCTION__, - __LINE__); - numberOfErrors += ViETest::TestError(codecObserver.outgoingRatecalled > 0, - "ERROR: %s at line %d", __FUNCTION__, - __LINE__); - //*************************************************************** - // Testing finished. Tear down Video Engine - //*************************************************************** - error = ptrViEBase->StopReceive(videoChannel); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViEBase->StopSend(videoChannel); // Already stopped - numberOfErrors += ViETest::TestError(error == -1, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViERender->StopRender(videoChannel); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViERender->RemoveRenderer(captureId); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViERender->RemoveRenderer(videoChannel); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViECapture->DisconnectCaptureDevice(videoChannel); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); - error = ptrViEBase->DeleteChannel(videoChannel); - numberOfErrors += ViETest::TestError(error == 0, "ERROR: %s at line %d", - __FUNCTION__, __LINE__); + error = render_interface->StopRender(video_channel); + number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + error = render_interface->RemoveRenderer(capture_id); + number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + error = render_interface->RemoveRenderer(video_channel); + number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + error = capture_interface->DisconnectCaptureDevice(video_channel); + number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); + error = base_interface->DeleteChannel(video_channel); + number_of_errors += ViETest::TestError(error == 0, "ERROR: %s at line %d", + __FUNCTION__, __LINE__); } diff --git a/src/video_engine/main/test/AutoTest/vie_auto_test.gypi b/src/video_engine/main/test/AutoTest/vie_auto_test.gypi index fd192d6e3c..a61e6ba614 100644 --- a/src/video_engine/main/test/AutoTest/vie_auto_test.gypi +++ b/src/video_engine/main/test/AutoTest/vie_auto_test.gypi @@ -18,6 +18,7 @@ '<(webrtc_root)/voice_engine/voice_engine.gyp:voice_engine_core', '<(webrtc_root)/../testing/gtest.gyp:gtest', '<(webrtc_root)/../third_party/google-gflags/google-gflags.gyp:google-gflags', + '<(webrtc_root)/../test/test.gyp:test_support', 'video_engine_core', ], 'include_dirs': [ @@ -42,12 +43,13 @@ 'interface/vie_autotest_main.h', 'interface/vie_autotest_window_manager_interface.h', 'interface/vie_autotest_windows.h', - + # Helper classes - 'helpers/vie_window_creator.cc', - 'helpers/vie_file_capture_device.cc', 'helpers/vie_fake_camera.cc', - + 'helpers/vie_file_capture_device.cc', + 'helpers/vie_to_file_renderer.cc', + 'helpers/vie_window_creator.cc', + # New, fully automated tests 'automated/vie_api_integration_test.cc', 'automated/vie_extended_integration_test.cc', diff --git a/test/test.gyp b/test/test.gyp index b010f1eb96..e01e0d9e8e 100644 --- a/test/test.gyp +++ b/test/test.gyp @@ -15,7 +15,13 @@ { 'target_name': 'test_support', 'type': 'static_library', + 'direct_dependent_settings': { + 'include_dirs': [ + '.', # Some includes are hierarchical + ], + }, 'dependencies': [ + '<(webrtc_root)/system_wrappers/source/system_wrappers.gyp:system_wrappers', '<(webrtc_root)/../testing/gtest.gyp:gtest', '<(webrtc_root)/../testing/gmock.gyp:gmock', ], @@ -29,6 +35,7 @@ 'test_suite.h', 'testsupport/fileutils.h', 'testsupport/fileutils.cc', + 'testsupport/metrics/video_metrics.cc', ], }, { diff --git a/src/modules/video_coding/main/test/video_metrics.cc b/test/testsupport/metrics/video_metrics.cc similarity index 98% rename from src/modules/video_coding/main/test/video_metrics.cc rename to test/testsupport/metrics/video_metrics.cc index cd03f02088..568c43ae3a 100644 --- a/src/modules/video_coding/main/test/video_metrics.cc +++ b/test/testsupport/metrics/video_metrics.cc @@ -85,10 +85,12 @@ PsnrFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName, refBytes = (WebRtc_Word32) fread(ref, 1, frameBytes, refFp); testBytes = (WebRtc_Word32) fread(test, 1, frameBytes, testFp); } - // for identical reproduction: + if (mse == 0) { - result->average = 48; + // The PSNR value is undefined in this case. + // This value effectively means that the files are equal. + result->average = std::numeric_limits::max(); } else { diff --git a/src/modules/video_coding/main/test/video_metrics.h b/test/testsupport/metrics/video_metrics.h similarity index 78% rename from src/modules/video_coding/main/test/video_metrics.h rename to test/testsupport/metrics/video_metrics.h index eecd271786..d25a1cfc57 100644 --- a/src/modules/video_coding/main/test/video_metrics.h +++ b/test/testsupport/metrics/video_metrics.h @@ -42,11 +42,20 @@ struct QualityMetricsResult { }; // PSNR & SSIM calculations + +// PSNR values are filled into the QualityMetricsResult struct. +// If the result is std::numerical_limits::max() the videos were +// equal. Otherwise, PSNR values are in decibel (higher is better). This +// algorithm only compares up to the point when the shortest video ends. WebRtc_Word32 PsnrFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName, WebRtc_Word32 width, WebRtc_Word32 height, QualityMetricsResult *result); + +// SSIM values are filled into the QualityMetricsResult struct. +// Values range between -1 and 1, where 1 means the files were identical. This +// algorithm only compares up to the point when the shortest video ends. WebRtc_Word32 SsimFromFiles(const WebRtc_Word8 *refFileName, const WebRtc_Word8 *testFileName, WebRtc_Word32 width,