Save encoded ivf files separately for different TLs.

This allows offline visualization of the different TL.

For now, there is no need to do the same for the decoded frames.

Bug: webrtc:10349
Tested: 1) ninja -C out/Debug; and out/Debug/modules_tests --gtest_filter="*MultiresVP8*:*SvcVP9*". 2) Downstream tests.
Change-Id: Iaf5ab19ee681488706d8777a5adba78efd5cc1ee
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/128861
Commit-Queue: Rasmus Brandt <brandtr@webrtc.org>
Reviewed-by: Mirta Dvornicic <mirtad@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#27240}

modules/video_coding/codecs/test/videocodec_test_fixture_impl.cc

@@ -648,18 +648,26 @@ void VideoCodecTestFixtureImpl::SetUpAndInitObjects(
   RTC_DCHECK(decoded_frame_writers_.empty());
   const size_t num_simulcast_or_spatial_layers = std::max(
       config_.NumberOfSimulcastStreams(), config_.NumberOfSpatialLayers());
+  const size_t num_temporal_layers = config_.NumberOfTemporalLayers();
   for (size_t simulcast_svc_idx = 0;
        simulcast_svc_idx < num_simulcast_or_spatial_layers;
        ++simulcast_svc_idx) {
-    const std::string output_filename_base = OutputPath() +
-                                             FilenameWithParams(config_) + "_" +
-                                             std::to_string(simulcast_svc_idx);
+    const std::string output_filename_base =
+        OutputPath() + FilenameWithParams(config_) + "_sl" +
+        std::to_string(simulcast_svc_idx);
 
     if (config_.visualization_params.save_encoded_ivf) {
-      FileWrapper post_encode_file =
-          FileWrapper::OpenWriteOnly(output_filename_base + ".ivf");
-      encoded_frame_writers_.push_back(
-          IvfFileWriter::Wrap(std::move(post_encode_file), 0));
+      for (size_t temporal_idx = 0; temporal_idx < num_temporal_layers;
+           ++temporal_idx) {
+        const std::string output_file_path =
+            output_filename_base + "tl" + std::to_string(temporal_idx) + ".ivf";
+        FileWrapper ivf_file = FileWrapper::OpenWriteOnly(output_file_path);
+
+        const VideoProcessor::LayerKey layer_key(simulcast_svc_idx,
+                                                 temporal_idx);
+        encoded_frame_writers_[layer_key] =
+            IvfFileWriter::Wrap(std::move(ivf_file), /*byte_limit=*/0);
+      }
     }
 
     if (config_.visualization_params.save_decoded_y4m) {
@@ -680,8 +688,7 @@ void VideoCodecTestFixtureImpl::SetUpAndInitObjects(
     CreateEncoderAndDecoder();
     processor_ = absl::make_unique<VideoProcessor>(
         encoder_.get(), &decoders_, source_frame_reader_.get(), config_,
-        &stats_,
-        encoded_frame_writers_.empty() ? nullptr : &encoded_frame_writers_,
+        &stats_, &encoded_frame_writers_,
         decoded_frame_writers_.empty() ? nullptr : &decoded_frame_writers_);
   });
 }
@@ -698,7 +705,7 @@ void VideoCodecTestFixtureImpl::ReleaseAndCloseObjects(
 
   // Close visualization files.
   for (auto& encoded_frame_writer : encoded_frame_writers_) {
-    EXPECT_TRUE(encoded_frame_writer->Close());
+    EXPECT_TRUE(encoded_frame_writer.second->Close());
   }
   encoded_frame_writers_.clear();
   for (auto& decoded_frame_writer : decoded_frame_writers_) {

modules/video_coding/codecs/test/videocodec_test_fixture_impl.h

@@ -95,7 +95,7 @@ class VideoCodecTestFixtureImpl : public VideoCodecTestFixture {
   Config config_;
   VideoCodecTestStatsImpl stats_;
   std::unique_ptr<FrameReader> source_frame_reader_;
-  VideoProcessor::IvfFileWriterList encoded_frame_writers_;
+  VideoProcessor::IvfFileWriterMap encoded_frame_writers_;
   VideoProcessor::FrameWriterList decoded_frame_writers_;
   std::unique_ptr<VideoProcessor> processor_;
   std::unique_ptr<CpuProcessTime> cpu_process_time_;

modules/video_coding/codecs/test/videoprocessor.cc

@@ -165,7 +165,7 @@ VideoProcessor::VideoProcessor(webrtc::VideoEncoder* encoder,
                                FrameReader* input_frame_reader,
                                const VideoCodecTestFixture::Config& config,
                                VideoCodecTestStats* stats,
-                               IvfFileWriterList* encoded_frame_writers,
+                               IvfFileWriterMap* encoded_frame_writers,
                                FrameWriterList* decoded_frame_writers)
     : config_(config),
       num_simulcast_or_spatial_layers_(
@@ -194,13 +194,12 @@ VideoProcessor::VideoProcessor(webrtc::VideoEncoder* encoder,
   // Sanity checks.
   RTC_CHECK(TaskQueueBase::Current())
       << "VideoProcessor must be run on a task queue.";
-  RTC_CHECK(encoder);
-  RTC_CHECK(decoders);
-  RTC_CHECK_EQ(decoders->size(), num_simulcast_or_spatial_layers_);
-  RTC_CHECK(input_frame_reader);
-  RTC_CHECK(stats);
-  RTC_CHECK(!encoded_frame_writers ||
-            encoded_frame_writers->size() == num_simulcast_or_spatial_layers_);
+  RTC_CHECK(stats_);
+  RTC_CHECK(encoder_);
+  RTC_CHECK(decoders_);
+  RTC_CHECK_EQ(decoders_->size(), num_simulcast_or_spatial_layers_);
+  RTC_CHECK(input_frame_reader_);
+  RTC_CHECK(encoded_frame_writers_);
   RTC_CHECK(!decoded_frame_writers ||
             decoded_frame_writers->size() == num_simulcast_or_spatial_layers_);
 
@@ -360,8 +359,8 @@ void VideoProcessor::FrameEncoded(
             last_encoded_frame_num_[spatial_idx] < frame_number);
 
   // Ensure SVC spatial layers are delivered in ascending order.
-  if (!first_encoded_frame_[spatial_idx] &&
-      config_.NumberOfSpatialLayers() > 1) {
+  const size_t num_spatial_layers = config_.NumberOfSpatialLayers();
+  if (!first_encoded_frame_[spatial_idx] && num_spatial_layers > 1) {
     for (size_t i = 0; i < spatial_idx; ++i) {
       RTC_CHECK_LE(last_encoded_frame_num_[i], frame_number);
     }
@@ -385,7 +384,6 @@ void VideoProcessor::FrameEncoded(
   frame_stat->max_nalu_size_bytes = GetMaxNaluSizeBytes(encoded_image, config_);
   frame_stat->qp = encoded_image.qp_;
 
-  const size_t num_spatial_layers = config_.NumberOfSpatialLayers();
   bool end_of_picture = false;
   if (codec_type == kVideoCodecVP9) {
     const CodecSpecificInfoVP9& vp9_info = codec_specific.codecSpecific.VP9;
@@ -399,7 +397,7 @@ void VideoProcessor::FrameEncoded(
   }
 
   const webrtc::EncodedImage* encoded_image_for_decode = &encoded_image;
-  if (config_.decode || encoded_frame_writers_) {
+  if (config_.decode || !encoded_frame_writers_->empty()) {
     if (num_spatial_layers > 1) {
       encoded_image_for_decode = BuildAndStoreSuperframe(
           encoded_image, codec_type, frame_number, spatial_idx,
@@ -436,10 +434,17 @@ void VideoProcessor::FrameEncoded(
     frame_stat->decode_return_code = WEBRTC_VIDEO_CODEC_NO_OUTPUT;
   }
 
-  if (encoded_frame_writers_) {
-    RTC_CHECK(encoded_frame_writers_->at(spatial_idx)
-                  ->WriteFrame(*encoded_image_for_decode,
-                               config_.codec_settings.codecType));
+  // Since frames in higher TLs typically depend on frames in lower TLs,
+  // write out frames in lower TLs to bitstream dumps of higher TLs.
+  for (size_t write_temporal_idx = temporal_idx;
+       write_temporal_idx < config_.NumberOfTemporalLayers();
+       ++write_temporal_idx) {
+    const VideoProcessor::LayerKey layer_key(spatial_idx, write_temporal_idx);
+    auto it = encoded_frame_writers_->find(layer_key);
+    if (it != encoded_frame_writers_->cend()) {
+      RTC_CHECK(it->second->WriteFrame(*encoded_image_for_decode,
+                                       config_.codec_settings.codecType));
+    }
   }
 
   if (!config_.encode_in_real_time) {

modules/video_coding/codecs/test/videoprocessor.h

@@ -15,6 +15,7 @@
 #include <stdint.h>
 #include <map>
 #include <memory>
+#include <utility>
 #include <vector>
 
 #include "absl/memory/memory.h"
@@ -53,7 +54,10 @@ namespace test {
 class VideoProcessor {
  public:
   using VideoDecoderList = std::vector<std::unique_ptr<VideoDecoder>>;
-  using IvfFileWriterList = std::vector<std::unique_ptr<IvfFileWriter>>;
+  using LayerKey = std::pair<int /* spatial_idx */, int /* temporal_idx */>;
+  using IvfFileWriterMap = std::map<LayerKey, std::unique_ptr<IvfFileWriter>>;
+  // TODO(brandtr): Consider changing FrameWriterList to be a FrameWriterMap,
+  // to be able to save different TLs separately.
   using FrameWriterList = std::vector<std::unique_ptr<FrameWriter>>;
 
   VideoProcessor(webrtc::VideoEncoder* encoder,
@@ -61,7 +65,7 @@ class VideoProcessor {
                  FrameReader* input_frame_reader,
                  const VideoCodecTestFixture::Config& config,
                  VideoCodecTestStats* stats,
-                 IvfFileWriterList* encoded_frame_writers,
+                 IvfFileWriterMap* encoded_frame_writers,
                  FrameWriterList* decoded_frame_writers);
   ~VideoProcessor();
 
@@ -220,7 +224,7 @@ class VideoProcessor {
 
   // These (optional) file writers are used to persistently store the encoded
   // and decoded bitstreams. Each frame writer is enabled by being non-null.
-  IvfFileWriterList* const encoded_frame_writers_;
+  IvfFileWriterMap* const encoded_frame_writers_;
   FrameWriterList* const decoded_frame_writers_;
 
   // Metadata for inputed/encoded/decoded frames. Used for frame identification,

modules/video_coding/codecs/test/videoprocessor_unittest.cc

@@ -54,8 +54,7 @@ class VideoProcessorTest : public testing::Test {
     q_.SendTask([this] {
       video_processor_ = absl::make_unique<VideoProcessor>(
           &encoder_mock_, &decoders_, &frame_reader_mock_, config_, &stats_,
-          nullptr /* encoded_frame_writer */,
-          nullptr /* decoded_frame_writer */);
+          &encoded_frame_writers_, /*decoded_frame_writers=*/nullptr);
     });
   }
 
@@ -86,6 +85,7 @@ class VideoProcessorTest : public testing::Test {
   std::vector<std::unique_ptr<VideoDecoder>> decoders_;
   MockFrameReader frame_reader_mock_;
   VideoCodecTestStatsImpl stats_;
+  VideoProcessor::IvfFileWriterMap encoded_frame_writers_;
   std::unique_ptr<VideoProcessor> video_processor_;
 };