Create and initialize encoders only for active streams

Bug: webrtc:12407
Change-Id: Id30fcb84dcbfffa30c7a34b15564ab5049cec96c
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/204066
Commit-Queue: Sergey Silkin <ssilkin@webrtc.org>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#33141}

media/BUILD.gn

@@ -576,6 +576,7 @@ if (rtc_include_tests) {
         "../api/units:time_delta",
         "../api/video:builtin_video_bitrate_allocator_factory",
         "../api/video:video_bitrate_allocation",
+        "../api/video:video_codec_constants",
         "../api/video:video_frame",
         "../api/video:video_rtp_headers",
         "../api/video_codecs:builtin_video_decoder_factory",

media/engine/encoder_simulcast_proxy_unittest.cc

@@ -49,7 +49,8 @@ TEST(EncoderSimulcastProxy, ChoosesCorrectImplementation) {
                                        2000,
                                        1000,
                                        1000,
-                                       56};
+                                       56,
+                                       true};
   codec_settings.simulcastStream[1] = {test::kTestWidth,
                                        test::kTestHeight,
                                        test::kTestFrameRate,
@@ -57,7 +58,8 @@ TEST(EncoderSimulcastProxy, ChoosesCorrectImplementation) {
                                        3000,
                                        1000,
                                        1000,
-                                       56};
+                                       56,
+                                       true};
   codec_settings.simulcastStream[2] = {test::kTestWidth,
                                        test::kTestHeight,
                                        test::kTestFrameRate,
@@ -65,7 +67,8 @@ TEST(EncoderSimulcastProxy, ChoosesCorrectImplementation) {
                                        5000,
                                        1000,
                                        1000,
-                                       56};
+                                       56,
+                                       true};
   codec_settings.numberOfSimulcastStreams = 3;
 
   auto mock_encoder = std::make_unique<NiceMock<MockVideoEncoder>>();

media/engine/simulcast_encoder_adapter.cc

@@ -62,32 +62,29 @@ uint32_t SumStreamMaxBitrate(int streams, const webrtc::VideoCodec& codec) {
   return bitrate_sum;
 }
 
-int NumberOfStreams(const webrtc::VideoCodec& codec) {
+int CountAllStreams(const webrtc::VideoCodec& codec) {
-  int streams =
+  int total_streams_count =
       codec.numberOfSimulcastStreams < 1 ? 1 : codec.numberOfSimulcastStreams;
-  uint32_t simulcast_max_bitrate = SumStreamMaxBitrate(streams, codec);
+  uint32_t simulcast_max_bitrate =
+      SumStreamMaxBitrate(total_streams_count, codec);
   if (simulcast_max_bitrate == 0) {
-    streams = 1;
+    total_streams_count = 1;
   }
-  return streams;
+  return total_streams_count;
 }
 
-struct StreamDimensions {
+int CountActiveStreams(const webrtc::VideoCodec& codec) {
-  size_t num_active_streams;
+  if (codec.numberOfSimulcastStreams < 1) {
-  size_t first_active_stream_idx;
+    return 1;
-};
+  }
-StreamDimensions ActiveStreams(const webrtc::VideoCodec& codec) {
+  int total_streams_count = CountAllStreams(codec);
-  size_t num_configured_streams = NumberOfStreams(codec);
+  int active_streams_count = 0;
-  StreamDimensions dimensions{0, 0};
+  for (int i = 0; i < total_streams_count; ++i) {
-  for (size_t i = 0; i < num_configured_streams; ++i) {
     if (codec.simulcastStream[i].active) {
-      ++dimensions.num_active_streams;
+      ++active_streams_count;
-      if (dimensions.num_active_streams == 1) {
-        dimensions.first_active_stream_idx = i;
     }
   }
-  }
+  return active_streams_count;
-  return dimensions;
 }
 
 int VerifyCodec(const webrtc::VideoCodec* inst) {
@@ -105,80 +102,119 @@ int VerifyCodec(const webrtc::VideoCodec* inst) {
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
   if (inst->codecType == webrtc::kVideoCodecVP8 &&
-      inst->VP8().automaticResizeOn &&
+      inst->VP8().automaticResizeOn && CountActiveStreams(*inst) > 1) {
-      ActiveStreams(*inst).num_active_streams > 1) {
     return WEBRTC_VIDEO_CODEC_ERR_PARAMETER;
   }
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
-bool StreamResolutionCompare(const webrtc::SpatialLayer& a,
+bool StreamQualityCompare(const webrtc::SpatialLayer& a,
                           const webrtc::SpatialLayer& b) {
   return std::tie(a.height, a.width, a.maxBitrate, a.maxFramerate) <
          std::tie(b.height, b.width, b.maxBitrate, b.maxFramerate);
 }
 
+void GetLowestAndHighestQualityStreamIndixes(
+    rtc::ArrayView<webrtc::SpatialLayer> streams,
+    int* lowest_quality_stream_idx,
+    int* highest_quality_stream_idx) {
+  const auto lowest_highest_quality_streams =
+      absl::c_minmax_element(streams, StreamQualityCompare);
+  *lowest_quality_stream_idx =
+      std::distance(streams.begin(), lowest_highest_quality_streams.first);
+  *highest_quality_stream_idx =
+      std::distance(streams.begin(), lowest_highest_quality_streams.second);
+}
+
+std::vector<uint32_t> GetStreamStartBitratesKbps(
+    const webrtc::VideoCodec& codec) {
+  std::vector<uint32_t> start_bitrates;
+  std::unique_ptr<webrtc::VideoBitrateAllocator> rate_allocator =
+      std::make_unique<webrtc::SimulcastRateAllocator>(codec);
+  webrtc::VideoBitrateAllocation allocation =
+      rate_allocator->Allocate(webrtc::VideoBitrateAllocationParameters(
+          codec.startBitrate * 1000, codec.maxFramerate));
+
+  int total_streams_count = CountAllStreams(codec);
+  for (int i = 0; i < total_streams_count; ++i) {
+    uint32_t stream_bitrate = allocation.GetSpatialLayerSum(i) / 1000;
+    start_bitrates.push_back(stream_bitrate);
+  }
+  return start_bitrates;
+}
+
 }  // namespace
 
 namespace webrtc {
 
 SimulcastEncoderAdapter::EncoderContext::EncoderContext(
-    SimulcastEncoderAdapter* parent,
     std::unique_ptr<VideoEncoder> encoder,
-    std::unique_ptr<FramerateController> framerate_controller,
+    bool prefer_temporal_support)
-    int stream_idx,
+    : encoder_(std::move(encoder)),
-    uint16_t width,
+      prefer_temporal_support_(prefer_temporal_support) {}
-    uint16_t height,
-    bool send_stream)
-    : parent_(parent),
-      encoder_(std::move(encoder)),
-      framerate_controller_(std::move(framerate_controller)),
-      stream_idx_(stream_idx),
-      width_(width),
-      height_(height),
-      needs_keyframe_(false),
-      send_stream_(send_stream) {
-  if (parent) {
-    encoder_->RegisterEncodeCompleteCallback(this);
-  }
-}
 
-SimulcastEncoderAdapter::EncoderContext::EncoderContext(EncoderContext&& rhs)
+void SimulcastEncoderAdapter::EncoderContext::Release() {
-    : parent_(rhs.parent_),
-      encoder_(std::move(rhs.encoder_)),
-      framerate_controller_(std::move(rhs.framerate_controller_)),
-      stream_idx_(rhs.stream_idx_),
-      width_(rhs.width_),
-      height_(rhs.height_),
-      needs_keyframe_(rhs.needs_keyframe_),
-      send_stream_(rhs.send_stream_) {
-  if (parent_) {
-    encoder_->RegisterEncodeCompleteCallback(this);
-  }
-}
-
-SimulcastEncoderAdapter::EncoderContext::~EncoderContext() {
   if (encoder_) {
     encoder_->RegisterEncodeCompleteCallback(nullptr);
     encoder_->Release();
   }
 }
 
-std::unique_ptr<VideoEncoder>
+SimulcastEncoderAdapter::StreamContext::StreamContext(
-SimulcastEncoderAdapter::EncoderContext::Release() && {
+    SimulcastEncoderAdapter* parent,
-  encoder_->RegisterEncodeCompleteCallback(nullptr);
+    std::unique_ptr<EncoderContext> encoder_context,
-  encoder_->Release();
+    std::unique_ptr<FramerateController> framerate_controller,
-  return std::move(encoder_);
+    int stream_idx,
+    uint16_t width,
+    uint16_t height,
+    bool is_paused)
+    : parent_(parent),
+      encoder_context_(std::move(encoder_context)),
+      framerate_controller_(std::move(framerate_controller)),
+      stream_idx_(stream_idx),
+      width_(width),
+      height_(height),
+      is_keyframe_needed_(false),
+      is_paused_(is_paused) {
+  if (parent_) {
+    encoder_context_->encoder().RegisterEncodeCompleteCallback(this);
+  }
 }
 
-void SimulcastEncoderAdapter::EncoderContext::OnKeyframe(Timestamp timestamp) {
+SimulcastEncoderAdapter::StreamContext::StreamContext(StreamContext&& rhs)
-  needs_keyframe_ = false;
+    : parent_(rhs.parent_),
+      encoder_context_(std::move(rhs.encoder_context_)),
+      framerate_controller_(std::move(rhs.framerate_controller_)),
+      stream_idx_(rhs.stream_idx_),
+      width_(rhs.width_),
+      height_(rhs.height_),
+      is_keyframe_needed_(rhs.is_keyframe_needed_),
+      is_paused_(rhs.is_paused_) {
+  if (parent_) {
+    encoder_context_->encoder().RegisterEncodeCompleteCallback(this);
+  }
+}
+
+SimulcastEncoderAdapter::StreamContext::~StreamContext() {
+  if (encoder_context_) {
+    encoder_context_->Release();
+  }
+}
+
+std::unique_ptr<SimulcastEncoderAdapter::EncoderContext>
+SimulcastEncoderAdapter::StreamContext::ReleaseEncoderContext() && {
+  encoder_context_->Release();
+  return std::move(encoder_context_);
+}
+
+void SimulcastEncoderAdapter::StreamContext::OnKeyframe(Timestamp timestamp) {
+  is_keyframe_needed_ = false;
   if (framerate_controller_) {
     framerate_controller_->AddFrame(timestamp.ms());
   }
 }
 
-bool SimulcastEncoderAdapter::EncoderContext::ShouldDropFrame(
+bool SimulcastEncoderAdapter::StreamContext::ShouldDropFrame(
     Timestamp timestamp) {
   if (!framerate_controller_) {
     return false;
@@ -192,7 +228,7 @@ bool SimulcastEncoderAdapter::EncoderContext::ShouldDropFrame(
 }
 
 EncodedImageCallback::Result
-SimulcastEncoderAdapter::EncoderContext::OnEncodedImage(
+SimulcastEncoderAdapter::StreamContext::OnEncodedImage(
     const EncodedImage& encoded_image,
     const CodecSpecificInfo* codec_specific_info) {
   RTC_CHECK(parent_);  // If null, this method should never be called.
@@ -200,7 +236,7 @@ SimulcastEncoderAdapter::EncoderContext::OnEncodedImage(
                                  codec_specific_info);
 }
 
-void SimulcastEncoderAdapter::EncoderContext::OnDroppedFrame(
+void SimulcastEncoderAdapter::StreamContext::OnDroppedFrame(
     DropReason /*reason*/) {
   RTC_CHECK(parent_);  // If null, this method should never be called.
   parent_->OnDroppedFrame(stream_idx_);
@@ -218,9 +254,9 @@ SimulcastEncoderAdapter::SimulcastEncoderAdapter(
       primary_encoder_factory_(primary_factory),
       fallback_encoder_factory_(fallback_factory),
       video_format_(format),
+      total_streams_count_(0),
+      bypass_mode_(false),
       encoded_complete_callback_(nullptr),
-      first_active_stream_idx_(0),
-      num_active_streams_(0),
       experimental_boosted_screenshare_qp_(GetScreenshareBoostedQpValue()),
       boost_base_layer_quality_(RateControlSettings::ParseFromFieldTrials()
                                     .Vp8BoostBaseLayerQuality()),
@@ -246,15 +282,15 @@ void SimulcastEncoderAdapter::SetFecControllerOverride(
 int SimulcastEncoderAdapter::Release() {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
 
-  while (!encoder_contexts_.empty()) {
+  while (!stream_contexts_.empty()) {
-    // Move the encoder instances and put it on the |stored_encoders_| where it
+    // Move the encoder instances and put it on the |cached_encoder_contexts_|
-    // it may possibly be reused from (ordering does not matter).
+    // where it may possibly be reused from (ordering does not matter).
-    stored_encoders_.push(std::move(encoder_contexts_.back()).Release());
+    cached_encoder_contexts_.push_front(
-    encoder_contexts_.pop_back();
+        std::move(stream_contexts_.back()).ReleaseEncoderContext());
+    stream_contexts_.pop_back();
   }
 
-  num_active_streams_ = 0;
+  bypass_mode_ = false;
-  first_active_stream_idx_ = 0;
 
   // It's legal to move the encoder to another queue now.
   encoder_queue_.Detach();
@@ -264,7 +300,6 @@ int SimulcastEncoderAdapter::Release() {
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
-// TODO(eladalon): s/inst/codec_settings/g.
 int SimulcastEncoderAdapter::InitEncode(
     const VideoCodec* inst,
     const VideoEncoder::Settings& settings) {
@@ -279,137 +314,114 @@ int SimulcastEncoderAdapter::InitEncode(
     return ret;
   }
 
-  ret = Release();
+  Release();
-  if (ret < 0) {
-    return ret;
-  }
-
-  int number_of_streams = NumberOfStreams(*inst);
-  RTC_DCHECK_LE(number_of_streams, kMaxSimulcastStreams);
-  bool doing_simulcast_using_adapter = (number_of_streams > 1);
-  auto active_streams = ActiveStreams(*inst);
-  num_active_streams_ = active_streams.num_active_streams;
-  first_active_stream_idx_ = active_streams.first_active_stream_idx;
 
   codec_ = *inst;
-  std::unique_ptr<VideoBitrateAllocator> rate_allocator =
+  total_streams_count_ = CountAllStreams(*inst);
-      std::make_unique<SimulcastRateAllocator>(codec_);
-
-  VideoBitrateAllocation allocation =
-      rate_allocator->Allocate(VideoBitrateAllocationParameters(
-          codec_.startBitrate * 1000, codec_.maxFramerate));
-  std::vector<uint32_t> start_bitrates;
-  for (int i = 0; i < kMaxSimulcastStreams; ++i) {
-    uint32_t stream_bitrate = allocation.GetSpatialLayerSum(i) / 1000;
-    start_bitrates.push_back(stream_bitrate);
-  }
-
-  // Create |number_of_streams| of encoder instances and init them.
-  auto spatial_layers =
-      rtc::ArrayView<SpatialLayer>(codec_.simulcastStream, number_of_streams);
-  const auto minmax =
-      absl::c_minmax_element(spatial_layers, StreamResolutionCompare);
-  const auto lowest_resolution_stream_index =
-      minmax.first - spatial_layers.begin();
-  const auto highest_resolution_stream_index =
-      minmax.second - spatial_layers.begin();
-
-  RTC_DCHECK_LT(lowest_resolution_stream_index, number_of_streams);
-  RTC_DCHECK_LT(highest_resolution_stream_index, number_of_streams);
-
-  for (int i = 0; i < number_of_streams; ++i) {
-    // If an existing encoder instance exists, reuse it.
-    // TODO(brandtr): Set initial RTP state (e.g., picture_id/tl0_pic_idx) here,
-    // when we start storing that state outside the encoder wrappers.
-    std::unique_ptr<VideoEncoder> encoder;
-    if (!stored_encoders_.empty()) {
-      encoder = std::move(stored_encoders_.top());
-      stored_encoders_.pop();
-    } else {
-      encoder = primary_encoder_factory_->CreateVideoEncoder(video_format_);
-      if (fallback_encoder_factory_ != nullptr) {
-        encoder = CreateVideoEncoderSoftwareFallbackWrapper(
-            fallback_encoder_factory_->CreateVideoEncoder(video_format_),
-            std::move(encoder),
-            i == lowest_resolution_stream_index &&
-                prefer_temporal_support_on_base_layer_);
-      }
-    }
-
-    bool encoder_initialized = false;
-    if (doing_simulcast_using_adapter && i == 0 &&
-        encoder->GetEncoderInfo().supports_simulcast) {
-      ret = encoder->InitEncode(&codec_, settings);
-      if (ret < 0) {
-        encoder->Release();
-      } else {
-        doing_simulcast_using_adapter = false;
-        number_of_streams = 1;
-        encoder_initialized = true;
-      }
-    }
-
-    VideoCodec stream_codec;
-    uint32_t start_bitrate_kbps = start_bitrates[i];
-    const bool send_stream = doing_simulcast_using_adapter
-                                 ? start_bitrate_kbps > 0
-                                 : num_active_streams_ > 0;
-    if (!doing_simulcast_using_adapter) {
-      stream_codec = codec_;
-      stream_codec.numberOfSimulcastStreams =
-          std::max<uint8_t>(1, stream_codec.numberOfSimulcastStreams);
-    } else {
-      // Cap start bitrate to the min bitrate in order to avoid strange codec
-      // behavior. Since sending will be false, this should not matter.
-      StreamResolution stream_resolution =
-          i == highest_resolution_stream_index
-              ? StreamResolution::HIGHEST
-              : i == lowest_resolution_stream_index ? StreamResolution::LOWEST
-                                                    : StreamResolution::OTHER;
-
-      start_bitrate_kbps =
-          std::max(spatial_layers[i].minBitrate, start_bitrate_kbps);
-      PopulateStreamCodec(codec_, i, start_bitrate_kbps, stream_resolution,
-                          &stream_codec);
-    }
 
   // TODO(ronghuawu): Remove once this is handled in LibvpxVp8Encoder.
-    if (stream_codec.qpMax < kDefaultMinQp) {
+  if (codec_.qpMax < kDefaultMinQp) {
-      stream_codec.qpMax = kDefaultMaxQp;
+    codec_.qpMax = kDefaultMaxQp;
   }
 
-    if (!encoder_initialized) {
+  bool is_legacy_singlecast = codec_.numberOfSimulcastStreams == 0;
-      ret = encoder->InitEncode(&stream_codec, settings);
+  int lowest_quality_stream_idx = 0;
+  int highest_quality_stream_idx = 0;
+  if (!is_legacy_singlecast) {
+    GetLowestAndHighestQualityStreamIndixes(
+        rtc::ArrayView<SpatialLayer>(codec_.simulcastStream,
+                                     total_streams_count_),
+        &lowest_quality_stream_idx, &highest_quality_stream_idx);
+  }
+
+  std::unique_ptr<EncoderContext> encoder_context = FetchOrCreateEncoderContext(
+      /*is_lowest_quality_stream=*/(
+          is_legacy_singlecast ||
+          codec_.simulcastStream[lowest_quality_stream_idx].active));
+  if (encoder_context == nullptr) {
+    return WEBRTC_VIDEO_CODEC_MEMORY;
+  }
+
+  // Two distinct scenarios:
+  // * Singlecast (total_streams_count == 1) or simulcast with simulcast-capable
+  //   underlaying encoder implementation. SEA operates in bypass mode: original
+  //   settings are passed to the underlaying encoder, frame encode complete
+  //   callback is not intercepted.
+  // * Multi-encoder simulcast or singlecast if layers are deactivated
+  //   (total_streams_count > 1 and active_streams_count >= 1). SEA creates
+  //   N=active_streams_count encoders and configures each to produce a single
+  //   stream.
+
+  // Singlecast or simulcast with simulcast-capable underlaying encoder.
+  if (total_streams_count_ == 1 ||
+      encoder_context->encoder().GetEncoderInfo().supports_simulcast) {
+    int ret = encoder_context->encoder().InitEncode(&codec_, settings);
+    if (ret >= 0) {
+      int active_streams_count = CountActiveStreams(*inst);
+      stream_contexts_.emplace_back(
+          /*parent=*/nullptr, std::move(encoder_context),
+          /*framerate_controller=*/nullptr, /*stream_idx=*/0, codec_.width,
+          codec_.height, /*is_paused=*/active_streams_count == 0);
+      bypass_mode_ = true;
+
+      DestroyStoredEncoders();
+      rtc::AtomicOps::ReleaseStore(&inited_, 1);
+      return WEBRTC_VIDEO_CODEC_OK;
+    }
+
+    encoder_context->Release();
+    if (total_streams_count_ == 1) {
+      // Failed to initialize singlecast encoder.
+      return ret;
+    }
+  }
+
+  // Multi-encoder simulcast or singlecast (deactivated layers).
+  std::vector<uint32_t> stream_start_bitrate_kbps =
+      GetStreamStartBitratesKbps(codec_);
+
+  for (int stream_idx = 0; stream_idx < total_streams_count_; ++stream_idx) {
+    if (!is_legacy_singlecast && !codec_.simulcastStream[stream_idx].active) {
+      continue;
+    }
+
+    if (encoder_context == nullptr) {
+      encoder_context = FetchOrCreateEncoderContext(
+          /*is_lowest_quality_stream=*/stream_idx == lowest_quality_stream_idx);
+    }
+    if (encoder_context == nullptr) {
+      Release();
+      return WEBRTC_VIDEO_CODEC_MEMORY;
+    }
+
+    VideoCodec stream_codec = MakeStreamCodec(
+        codec_, stream_idx, stream_start_bitrate_kbps[stream_idx],
+        /*is_lowest_quality_stream=*/stream_idx == lowest_quality_stream_idx,
+        /*is_highest_quality_stream=*/stream_idx == highest_quality_stream_idx);
+
+    int ret = encoder_context->encoder().InitEncode(&stream_codec, settings);
     if (ret < 0) {
-        // Explicitly destroy the current encoder; because we haven't registered
+      encoder_context.reset();
-        // a StreamInfo for it yet, Release won't do anything about it.
-        encoder.reset();
       Release();
       return ret;
     }
-    }
 
-    if (!doing_simulcast_using_adapter) {
+    // Intercept frame encode complete callback only for upper streams, where
-      // Without simulcast, let the encoder call callbacks and do frame
+    // we need to set a correct stream index. Set |parent| to nullptr for the
-      // dropping directly, without delegating to this adapter.
+    // lowest stream to bypass the callback.
-      encoder->RegisterEncodeCompleteCallback(encoded_complete_callback_);
+    SimulcastEncoderAdapter* parent = stream_idx > 0 ? this : nullptr;
-      encoder_contexts_.emplace_back(
+
-          /*parent=*/nullptr, std::move(encoder),
+    bool is_paused = stream_start_bitrate_kbps[stream_idx] == 0;
-          /*framerate_controller=*/nullptr, /*stream_idx=*/0,
+    stream_contexts_.emplace_back(
-          stream_codec.width, stream_codec.height, send_stream);
+        parent, std::move(encoder_context),
-      break;
-    }
-    encoder_contexts_.emplace_back(
-        this, std::move(encoder),
         std::make_unique<FramerateController>(stream_codec.maxFramerate),
-        /*stream_idx=*/i, stream_codec.width, stream_codec.height, send_stream);
+        stream_idx, stream_codec.width, stream_codec.height, is_paused);
   }
 
   // To save memory, don't store encoders that we don't use.
   DestroyStoredEncoders();
 
   rtc::AtomicOps::ReleaseStore(&inited_, 1);
-
   return WEBRTC_VIDEO_CODEC_OK;
 }
 
@@ -436,7 +448,7 @@ int SimulcastEncoderAdapter::Encode(
       return WEBRTC_VIDEO_CODEC_ERROR;
     }
     if (encoder_info_override_.apply_alignment_to_all_simulcast_layers()) {
-      for (const auto& layer : encoder_contexts_) {
+      for (const auto& layer : stream_contexts_) {
         if (layer.width() % alignment != 0 || layer.height() % alignment != 0) {
           RTC_LOG(LS_WARNING)
               << "Codec " << layer.width() << "x" << layer.height()
@@ -449,31 +461,33 @@ int SimulcastEncoderAdapter::Encode(
 
   // All active streams should generate a key frame if
   // a key frame is requested by any stream.
-  bool send_key_frame = false;
+  bool is_keyframe_needed = false;
   if (frame_types) {
-    for (size_t i = 0; i < frame_types->size(); ++i) {
+    for (const auto& frame_type : *frame_types) {
-      if (frame_types->at(i) == VideoFrameType::kVideoFrameKey) {
+      if (frame_type == VideoFrameType::kVideoFrameKey) {
-        send_key_frame = true;
+        is_keyframe_needed = true;
         break;
       }
     }
   }
 
-  for (const auto& layer : encoder_contexts_) {
+  if (!is_keyframe_needed) {
-    if (layer.needs_keyframe()) {
+    for (const auto& layer : stream_contexts_) {
-      send_key_frame = true;
+      if (layer.is_keyframe_needed()) {
+        is_keyframe_needed = true;
         break;
       }
     }
+  }
 
   // Temporary thay may hold the result of texture to i420 buffer conversion.
   rtc::scoped_refptr<VideoFrameBuffer> src_buffer;
   int src_width = input_image.width();
   int src_height = input_image.height();
 
-  for (auto& layer : encoder_contexts_) {
+  for (auto& layer : stream_contexts_) {
     // Don't encode frames in resolutions that we don't intend to send.
-    if (!layer.send_stream()) {
+    if (layer.is_paused()) {
       continue;
     }
 
@@ -485,8 +499,8 @@ int SimulcastEncoderAdapter::Encode(
     // frame types for all streams should be passed to the encoder unchanged.
     // Otherwise a single per-encoder frame type is passed.
     std::vector<VideoFrameType> stream_frame_types(
-        encoder_contexts_.size() == 1 ? NumberOfStreams(codec_) : 1);
+        bypass_mode_ ? total_streams_count_ : 1);
-    if (send_key_frame) {
+    if (is_keyframe_needed) {
       std::fill(stream_frame_types.begin(), stream_frame_types.end(),
                 VideoFrameType::kVideoFrameKey);
       layer.OnKeyframe(frame_timestamp);
@@ -548,9 +562,10 @@ int SimulcastEncoderAdapter::RegisterEncodeCompleteCallback(
     EncodedImageCallback* callback) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
   encoded_complete_callback_ = callback;
-  if (encoder_contexts_.size() == 1) {
+  if (!stream_contexts_.empty() && stream_contexts_.front().stream_idx() == 0) {
-    encoder_contexts_.front().encoder().RegisterEncodeCompleteCallback(
+    // Bypass frame encode complete callback for the lowest layer since there is
-        callback);
+    // no need to override frame's spatial index.
+    stream_contexts_.front().encoder().RegisterEncodeCompleteCallback(callback);
   }
   return WEBRTC_VIDEO_CODEC_OK;
 }
@@ -571,31 +586,21 @@ void SimulcastEncoderAdapter::SetRates(
 
   codec_.maxFramerate = static_cast<uint32_t>(parameters.framerate_fps + 0.5);
 
-  if (encoder_contexts_.size() == 1) {
+  if (bypass_mode_) {
-    // Not doing simulcast.
+    stream_contexts_.front().encoder().SetRates(parameters);
-    encoder_contexts_.front().encoder().SetRates(parameters);
     return;
   }
 
-  num_active_streams_ = 0;
+  for (StreamContext& layer_context : stream_contexts_) {
-  first_active_stream_idx_ = 0;
+    int stream_idx = layer_context.stream_idx();
-  for (size_t stream_idx = 0; stream_idx < encoder_contexts_.size();
-       ++stream_idx) {
-    EncoderContext& layer = encoder_contexts_[stream_idx];
     uint32_t stream_bitrate_kbps =
         parameters.bitrate.GetSpatialLayerSum(stream_idx) / 1000;
-    if (stream_bitrate_kbps > 0) {
-      if (num_active_streams_ == 0) {
-        first_active_stream_idx_ = stream_idx;
-      }
-      ++num_active_streams_;
-    }
 
     // Need a key frame if we have not sent this stream before.
-    if (stream_bitrate_kbps > 0 && !layer.send_stream()) {
+    if (stream_bitrate_kbps > 0 && layer_context.is_paused()) {
-      layer.set_keyframe_needed();
+      layer_context.set_is_keyframe_needed();
     }
-    layer.set_send_stream(stream_bitrate_kbps > 0);
+    layer_context.set_is_paused(stream_bitrate_kbps == 0);
 
     // Slice the temporal layers out of the full allocation and pass it on to
     // the encoder handling the current simulcast stream.
@@ -623,29 +628,29 @@ void SimulcastEncoderAdapter::SetRates(
       }
     }
 
-    stream_parameters.framerate_fps =
+    stream_parameters.framerate_fps = std::min<double>(
-        std::min<double>(parameters.framerate_fps,
+        parameters.framerate_fps,
-                         layer.target_fps().value_or(parameters.framerate_fps));
+        layer_context.target_fps().value_or(parameters.framerate_fps));
 
-    layer.encoder().SetRates(stream_parameters);
+    layer_context.encoder().SetRates(stream_parameters);
   }
 }
 
 void SimulcastEncoderAdapter::OnPacketLossRateUpdate(float packet_loss_rate) {
-  for (auto& c : encoder_contexts_) {
+  for (auto& c : stream_contexts_) {
     c.encoder().OnPacketLossRateUpdate(packet_loss_rate);
   }
 }
 
 void SimulcastEncoderAdapter::OnRttUpdate(int64_t rtt_ms) {
-  for (auto& c : encoder_contexts_) {
+  for (auto& c : stream_contexts_) {
     c.encoder().OnRttUpdate(rtt_ms);
   }
 }
 
 void SimulcastEncoderAdapter::OnLossNotification(
     const LossNotification& loss_notification) {
-  for (auto& c : encoder_contexts_) {
+  for (auto& c : stream_contexts_) {
     c.encoder().OnLossNotification(loss_notification);
   }
 }
@@ -669,72 +674,114 @@ void SimulcastEncoderAdapter::OnDroppedFrame(size_t stream_idx) {
   // Not yet implemented.
 }
 
-void SimulcastEncoderAdapter::PopulateStreamCodec(
-    const webrtc::VideoCodec& inst,
-    int stream_index,
-    uint32_t start_bitrate_kbps,
-    StreamResolution stream_resolution,
-    webrtc::VideoCodec* stream_codec) {
-  *stream_codec = inst;
-
-  // Stream specific simulcast settings.
-  const SpatialLayer* spatial_layers = inst.simulcastStream;
-
-  stream_codec->numberOfSimulcastStreams = 0;
-  stream_codec->width = spatial_layers[stream_index].width;
-  stream_codec->height = spatial_layers[stream_index].height;
-  stream_codec->maxBitrate = spatial_layers[stream_index].maxBitrate;
-  stream_codec->minBitrate = spatial_layers[stream_index].minBitrate;
-  stream_codec->maxFramerate = spatial_layers[stream_index].maxFramerate;
-  stream_codec->qpMax = spatial_layers[stream_index].qpMax;
-  stream_codec->active = spatial_layers[stream_index].active;
-  // Settings that are based on stream/resolution.
-  if (stream_resolution == StreamResolution::LOWEST) {
-    // Settings for lowest spatial resolutions.
-    if (inst.mode == VideoCodecMode::kScreensharing) {
-      if (experimental_boosted_screenshare_qp_) {
-        stream_codec->qpMax = *experimental_boosted_screenshare_qp_;
-      }
-    } else if (boost_base_layer_quality_) {
-      stream_codec->qpMax = kLowestResMaxQp;
-    }
-  }
-  if (inst.codecType == webrtc::kVideoCodecVP8) {
-    stream_codec->VP8()->numberOfTemporalLayers =
-        spatial_layers[stream_index].numberOfTemporalLayers;
-    if (stream_resolution != StreamResolution::HIGHEST) {
-      // For resolutions below CIF, set the codec |complexity| parameter to
-      // kComplexityHigher, which maps to cpu_used = -4.
-      int pixels_per_frame = stream_codec->width * stream_codec->height;
-      if (pixels_per_frame < 352 * 288) {
-        stream_codec->VP8()->complexity =
-            webrtc::VideoCodecComplexity::kComplexityHigher;
-      }
-      // Turn off denoising for all streams but the highest resolution.
-      stream_codec->VP8()->denoisingOn = false;
-    }
-  } else if (inst.codecType == webrtc::kVideoCodecH264) {
-    stream_codec->H264()->numberOfTemporalLayers =
-        spatial_layers[stream_index].numberOfTemporalLayers;
-  }
-
-  stream_codec->startBitrate = start_bitrate_kbps;
-
-  // Legacy screenshare mode is only enabled for the first simulcast layer
-  stream_codec->legacy_conference_mode =
-      inst.legacy_conference_mode && stream_index == 0;
-}
-
 bool SimulcastEncoderAdapter::Initialized() const {
   return rtc::AtomicOps::AcquireLoad(&inited_) == 1;
 }
 
 void SimulcastEncoderAdapter::DestroyStoredEncoders() {
-  while (!stored_encoders_.empty()) {
+  while (!cached_encoder_contexts_.empty()) {
-    stored_encoders_.pop();
+    cached_encoder_contexts_.pop_back();
   }
 }
 
+std::unique_ptr<SimulcastEncoderAdapter::EncoderContext>
+SimulcastEncoderAdapter::FetchOrCreateEncoderContext(
+    bool is_lowest_quality_stream) {
+  bool prefer_temporal_support = fallback_encoder_factory_ != nullptr &&
+                                 is_lowest_quality_stream &&
+                                 prefer_temporal_support_on_base_layer_;
+
+  // Toggling of |prefer_temporal_support| requires encoder recreation. Find
+  // and reuse encoder with desired |prefer_temporal_support|. Otherwise, if
+  // there is no such encoder in the cache, create a new instance.
+  auto encoder_context_iter =
+      std::find_if(cached_encoder_contexts_.begin(),
+                   cached_encoder_contexts_.end(), [&](auto& encoder_context) {
+                     return encoder_context->prefer_temporal_support() ==
+                            prefer_temporal_support;
+                   });
+
+  std::unique_ptr<SimulcastEncoderAdapter::EncoderContext> encoder_context;
+  if (encoder_context_iter != cached_encoder_contexts_.end()) {
+    encoder_context = std::move(*encoder_context_iter);
+    cached_encoder_contexts_.erase(encoder_context_iter);
+  } else {
+    std::unique_ptr<VideoEncoder> encoder =
+        primary_encoder_factory_->CreateVideoEncoder(video_format_);
+    if (fallback_encoder_factory_ != nullptr) {
+      encoder = CreateVideoEncoderSoftwareFallbackWrapper(
+          fallback_encoder_factory_->CreateVideoEncoder(video_format_),
+          std::move(encoder), prefer_temporal_support);
+    }
+
+    encoder_context = std::make_unique<SimulcastEncoderAdapter::EncoderContext>(
+        std::move(encoder), prefer_temporal_support);
+  }
+
+  encoder_context->encoder().RegisterEncodeCompleteCallback(
+      encoded_complete_callback_);
+  return encoder_context;
+}
+
+webrtc::VideoCodec SimulcastEncoderAdapter::MakeStreamCodec(
+    const webrtc::VideoCodec& codec,
+    int stream_idx,
+    uint32_t start_bitrate_kbps,
+    bool is_lowest_quality_stream,
+    bool is_highest_quality_stream) {
+  webrtc::VideoCodec codec_params = codec;
+  const SpatialLayer& stream_params = codec.simulcastStream[stream_idx];
+
+  codec_params.numberOfSimulcastStreams = 0;
+  codec_params.width = stream_params.width;
+  codec_params.height = stream_params.height;
+  codec_params.maxBitrate = stream_params.maxBitrate;
+  codec_params.minBitrate = stream_params.minBitrate;
+  codec_params.maxFramerate = stream_params.maxFramerate;
+  codec_params.qpMax = stream_params.qpMax;
+  codec_params.active = stream_params.active;
+  // Settings that are based on stream/resolution.
+  if (is_lowest_quality_stream) {
+    // Settings for lowest spatial resolutions.
+    if (codec.mode == VideoCodecMode::kScreensharing) {
+      if (experimental_boosted_screenshare_qp_) {
+        codec_params.qpMax = *experimental_boosted_screenshare_qp_;
+      }
+    } else if (boost_base_layer_quality_) {
+      codec_params.qpMax = kLowestResMaxQp;
+    }
+  }
+  if (codec.codecType == webrtc::kVideoCodecVP8) {
+    codec_params.VP8()->numberOfTemporalLayers =
+        stream_params.numberOfTemporalLayers;
+    if (!is_highest_quality_stream) {
+      // For resolutions below CIF, set the codec |complexity| parameter to
+      // kComplexityHigher, which maps to cpu_used = -4.
+      int pixels_per_frame = codec_params.width * codec_params.height;
+      if (pixels_per_frame < 352 * 288) {
+        codec_params.VP8()->complexity =
+            webrtc::VideoCodecComplexity::kComplexityHigher;
+      }
+      // Turn off denoising for all streams but the highest resolution.
+      codec_params.VP8()->denoisingOn = false;
+    }
+  } else if (codec.codecType == webrtc::kVideoCodecH264) {
+    codec_params.H264()->numberOfTemporalLayers =
+        stream_params.numberOfTemporalLayers;
+  }
+
+  // Cap start bitrate to the min bitrate in order to avoid strange codec
+  // behavior.
+  codec_params.startBitrate =
+      std::max(stream_params.minBitrate, start_bitrate_kbps);
+
+  // Legacy screenshare mode is only enabled for the first simulcast layer
+  codec_params.legacy_conference_mode =
+      codec.legacy_conference_mode && stream_idx == 0;
+
+  return codec_params;
+}
+
 void SimulcastEncoderAdapter::OverrideFromFieldTrial(
     VideoEncoder::EncoderInfo* info) const {
   if (encoder_info_override_.requested_resolution_alignment()) {
@@ -750,10 +797,10 @@ void SimulcastEncoderAdapter::OverrideFromFieldTrial(
 }
 
 VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const {
-  if (encoder_contexts_.size() == 1) {
+  if (stream_contexts_.size() == 1) {
     // Not using simulcast adapting functionality, just pass through.
     VideoEncoder::EncoderInfo info =
-        encoder_contexts_.front().encoder().GetEncoderInfo();
+        stream_contexts_.front().encoder().GetEncoderInfo();
     OverrideFromFieldTrial(&info);
     return info;
   }
@@ -764,17 +811,16 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const {
   encoder_info.apply_alignment_to_all_simulcast_layers = false;
   encoder_info.supports_native_handle = true;
   encoder_info.scaling_settings.thresholds = absl::nullopt;
-  if (encoder_contexts_.empty()) {
+  if (stream_contexts_.empty()) {
     OverrideFromFieldTrial(&encoder_info);
     return encoder_info;
   }
 
   encoder_info.scaling_settings = VideoEncoder::ScalingSettings::kOff;
-  auto active_streams = ActiveStreams(codec_);
 
-  for (size_t i = 0; i < encoder_contexts_.size(); ++i) {
+  for (size_t i = 0; i < stream_contexts_.size(); ++i) {
     VideoEncoder::EncoderInfo encoder_impl_info =
-        encoder_contexts_[i].encoder().GetEncoderInfo();
+        stream_contexts_[i].encoder().GetEncoderInfo();
 
     if (i == 0) {
       // Encoder name indicates names of all sub-encoders.
@@ -817,10 +863,6 @@ VideoEncoder::EncoderInfo SimulcastEncoderAdapter::GetEncoderInfo() const {
     if (encoder_impl_info.apply_alignment_to_all_simulcast_layers) {
       encoder_info.apply_alignment_to_all_simulcast_layers = true;
     }
-    if (active_streams.num_active_streams == 1 &&
-        codec_.simulcastStream[i].active) {
-      encoder_info.scaling_settings = encoder_impl_info.scaling_settings;
-    }
   }
   encoder_info.implementation_name += ")";
 

media/engine/simulcast_encoder_adapter.h

@@ -12,6 +12,7 @@
 #ifndef MEDIA_ENGINE_SIMULCAST_ENCODER_ADAPTER_H_
 #define MEDIA_ENGINE_SIMULCAST_ENCODER_ADAPTER_H_
 
+#include <list>
 #include <memory>
 #include <stack>
 #include <string>
@@ -67,32 +68,50 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder {
   EncoderInfo GetEncoderInfo() const override;
 
  private:
-  class EncoderContext : public EncodedImageCallback {
+  class EncoderContext {
    public:
-    EncoderContext(SimulcastEncoderAdapter* parent,
+    EncoderContext(std::unique_ptr<VideoEncoder> encoder,
-                   std::unique_ptr<VideoEncoder> encoder,
+                   bool prefer_temporal_support);
+    EncoderContext& operator=(EncoderContext&&) = delete;
+
+    VideoEncoder& encoder() { return *encoder_; }
+    bool prefer_temporal_support() { return prefer_temporal_support_; }
+    void Release();
+
+   private:
+    std::unique_ptr<VideoEncoder> encoder_;
+    bool prefer_temporal_support_;
+  };
+
+  class StreamContext : public EncodedImageCallback {
+   public:
+    StreamContext(SimulcastEncoderAdapter* parent,
+                  std::unique_ptr<EncoderContext> encoder_context,
                   std::unique_ptr<FramerateController> framerate_controller,
                   int stream_idx,
                   uint16_t width,
                   uint16_t height,
                   bool send_stream);
-    EncoderContext(EncoderContext&& rhs);
+    StreamContext(StreamContext&& rhs);
-    EncoderContext& operator=(EncoderContext&&) = delete;
+    StreamContext& operator=(StreamContext&&) = delete;
-    ~EncoderContext() override;
+    ~StreamContext() override;
 
     Result OnEncodedImage(
         const EncodedImage& encoded_image,
         const CodecSpecificInfo* codec_specific_info) override;
     void OnDroppedFrame(DropReason reason) override;
 
-    VideoEncoder& encoder() { return *encoder_; }
+    VideoEncoder& encoder() { return encoder_context_->encoder(); }
-    const VideoEncoder& encoder() const { return *encoder_; }
+    const VideoEncoder& encoder() const { return encoder_context_->encoder(); }
+    int stream_idx() const { return stream_idx_; }
     uint16_t width() const { return width_; }
     uint16_t height() const { return height_; }
-    bool needs_keyframe() const { return send_stream_ && needs_keyframe_; }
+    bool is_keyframe_needed() const {
-    void set_keyframe_needed() { needs_keyframe_ = true; }
+      return !is_paused_ && is_keyframe_needed_;
-    bool send_stream() const { return send_stream_; }
+    }
-    void set_send_stream(bool send_stream) { send_stream_ = send_stream; }
+    void set_is_keyframe_needed() { is_keyframe_needed_ = true; }
+    bool is_paused() const { return is_paused_; }
+    void set_is_paused(bool is_paused) { is_paused_ = is_paused; }
     absl::optional<float> target_fps() const {
       return framerate_controller_ == nullptr
                  ? absl::nullopt
@@ -100,38 +119,34 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder {
                        framerate_controller_->GetTargetRate());
     }
 
-    std::unique_ptr<VideoEncoder> Release() &&;
+    std::unique_ptr<EncoderContext> ReleaseEncoderContext() &&;
     void OnKeyframe(Timestamp timestamp);
     bool ShouldDropFrame(Timestamp timestamp);
 
    private:
     SimulcastEncoderAdapter* const parent_;
-    std::unique_ptr<VideoEncoder> encoder_;
+    std::unique_ptr<EncoderContext> encoder_context_;
     std::unique_ptr<FramerateController> framerate_controller_;
     const int stream_idx_;
     const uint16_t width_;
     const uint16_t height_;
-    bool needs_keyframe_;
+    bool is_keyframe_needed_;
-    bool send_stream_;
+    bool is_paused_;
   };
 
-  enum class StreamResolution {
-    OTHER,
-    HIGHEST,
-    LOWEST,
-  };
-
-  // Populate the codec settings for each simulcast stream.
-  void PopulateStreamCodec(const webrtc::VideoCodec& inst,
-                           int stream_index,
-                           uint32_t start_bitrate_kbps,
-                           StreamResolution stream_resolution,
-                           webrtc::VideoCodec* stream_codec);
-
   bool Initialized() const;
 
   void DestroyStoredEncoders();
 
+  std::unique_ptr<EncoderContext> FetchOrCreateEncoderContext(
+      bool is_lowest_quality_stream);
+
+  webrtc::VideoCodec MakeStreamCodec(const webrtc::VideoCodec& codec,
+                                     int stream_idx,
+                                     uint32_t start_bitrate_kbps,
+                                     bool is_lowest_quality_stream,
+                                     bool is_highest_quality_stream);
+
   EncodedImageCallback::Result OnEncodedImage(
       size_t stream_idx,
       const EncodedImage& encoded_image,
@@ -146,17 +161,17 @@ class RTC_EXPORT SimulcastEncoderAdapter : public VideoEncoder {
   VideoEncoderFactory* const fallback_encoder_factory_;
   const SdpVideoFormat video_format_;
   VideoCodec codec_;
-  std::vector<EncoderContext> encoder_contexts_;
+  int total_streams_count_;
+  bool bypass_mode_;
+  std::vector<StreamContext> stream_contexts_;
   EncodedImageCallback* encoded_complete_callback_;
-  size_t first_active_stream_idx_;
-  size_t num_active_streams_;
 
   // Used for checking the single-threaded access of the encoder interface.
   RTC_NO_UNIQUE_ADDRESS SequenceChecker encoder_queue_;
 
   // Store encoders in between calls to Release and InitEncode, so they don't
   // have to be recreated. Remaining encoders are destroyed by the destructor.
-  std::stack<std::unique_ptr<VideoEncoder>> stored_encoders_;
+  std::list<std::unique_ptr<EncoderContext>> cached_encoder_contexts_;
 
   const absl::optional<unsigned int> experimental_boosted_screenshare_qp_;
   const bool boost_base_layer_quality_;

media/engine/simulcast_encoder_adapter_unittest.cc

@@ -18,6 +18,7 @@
 #include "api/test/simulcast_test_fixture.h"
 #include "api/test/video/function_video_decoder_factory.h"
 #include "api/test/video/function_video_encoder_factory.h"
+#include "api/video/video_codec_constants.h"
 #include "api/video_codecs/sdp_video_format.h"
 #include "api/video_codecs/video_encoder.h"
 #include "api/video_codecs/video_encoder_factory.h"
@@ -421,14 +422,24 @@ class TestSimulcastEncoderAdapterFake : public ::testing::Test,
   }
 
   void SetUp() override {
-    helper_ = std::make_unique<TestSimulcastEncoderAdapterFakeHelper>(
+    helper_.reset(new TestSimulcastEncoderAdapterFakeHelper(
-        use_fallback_factory_, SdpVideoFormat("VP8", sdp_video_parameters_));
+        use_fallback_factory_, SdpVideoFormat("VP8", sdp_video_parameters_)));
     adapter_.reset(helper_->CreateMockEncoderAdapter());
     last_encoded_image_width_ = -1;
     last_encoded_image_height_ = -1;
     last_encoded_image_simulcast_index_ = -1;
   }
 
+  void ReSetUp() {
+    if (adapter_) {
+      adapter_->Release();
+      // |helper_| owns factories which |adapter_| needs to destroy encoders.
+      // Release |adapter_| before |helper_| (released in SetUp()).
+      adapter_.reset();
+    }
+    SetUp();
+  }
+
   Result OnEncodedImage(const EncodedImage& encoded_image,
                         const CodecSpecificInfo* codec_specific_info) override {
     last_encoded_image_width_ = encoded_image._encodedWidth;
@@ -451,10 +462,23 @@ class TestSimulcastEncoderAdapterFake : public ::testing::Test,
     return true;
   }
 
-  void SetupCodec() {
+  void SetupCodec() { SetupCodec(/*active_streams=*/{true, true, true}); }
+
+  void SetupCodec(std::vector<bool> active_streams) {
     SimulcastTestFixtureImpl::DefaultSettings(
         &codec_, static_cast<const int*>(kTestTemporalLayerProfile),
         kVideoCodecVP8);
+    ASSERT_LE(active_streams.size(), codec_.numberOfSimulcastStreams);
+    codec_.numberOfSimulcastStreams = active_streams.size();
+    for (size_t stream_idx = 0; stream_idx < kMaxSimulcastStreams;
+         ++stream_idx) {
+      if (stream_idx >= codec_.numberOfSimulcastStreams) {
+        // Reset parameters of unspecified stream.
+        codec_.simulcastStream[stream_idx] = {0};
+      } else {
+        codec_.simulcastStream[stream_idx].active = active_streams[stream_idx];
+      }
+    }
     rate_allocator_.reset(new SimulcastRateAllocator(codec_));
     EXPECT_EQ(0, adapter_->InitEncode(&codec_, kSettings));
     adapter_->RegisterEncodeCompleteCallback(this);
@@ -579,7 +603,8 @@ TEST_F(TestSimulcastEncoderAdapterFake, EncodedCallbackForDifferentEncoders) {
   EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index));
   EXPECT_EQ(1152, width);
   EXPECT_EQ(704, height);
-  EXPECT_EQ(0, simulcast_index);
+  // SEA doesn't intercept frame encode complete callback for the lowest stream.
+  EXPECT_EQ(-1, simulcast_index);
 
   encoders[1]->SendEncodedImage(300, 620);
   EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index));
@@ -795,7 +820,8 @@ TEST_F(TestSimulcastEncoderAdapterFake, ReinitDoesNotReorderFrameSimulcastIdx) {
   int height;
   int simulcast_index;
   EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index));
-  EXPECT_EQ(0, simulcast_index);
+  // SEA doesn't intercept frame encode complete callback for the lowest stream.
+  EXPECT_EQ(-1, simulcast_index);
 
   encoders[1]->SendEncodedImage(300, 620);
   EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index));
@@ -815,7 +841,7 @@ TEST_F(TestSimulcastEncoderAdapterFake, ReinitDoesNotReorderFrameSimulcastIdx) {
   // Verify that the same encoder sends out frames on the same simulcast index.
   encoders[0]->SendEncodedImage(1152, 704);
   EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index));
-  EXPECT_EQ(0, simulcast_index);
+  EXPECT_EQ(-1, simulcast_index);
 
   encoders[1]->SendEncodedImage(300, 620);
   EXPECT_TRUE(GetLastEncodedImageInfo(&width, &height, &simulcast_index));
@@ -1593,5 +1619,69 @@ TEST_F(TestSimulcastEncoderAdapterFake, SupportsPerSimulcastLayerMaxFramerate) {
   EXPECT_EQ(10u, helper_->factory()->encoders()[2]->codec().maxFramerate);
 }
 
+TEST_F(TestSimulcastEncoderAdapterFake, CreatesEncoderOnlyIfStreamIsActive) {
+  // Legacy singlecast
+  SetupCodec(/*active_streams=*/{});
+  EXPECT_EQ(1u, helper_->factory()->encoders().size());
+
+  // Simulcast-capable underlaying encoder
+  ReSetUp();
+  helper_->factory()->set_supports_simulcast(true);
+  SetupCodec(/*active_streams=*/{true, true});
+  EXPECT_EQ(1u, helper_->factory()->encoders().size());
+
+  // Muti-encoder simulcast
+  ReSetUp();
+  helper_->factory()->set_supports_simulcast(false);
+  SetupCodec(/*active_streams=*/{true, true});
+  EXPECT_EQ(2u, helper_->factory()->encoders().size());
+
+  // Singlecast via layers deactivation. Lowest layer is active.
+  ReSetUp();
+  helper_->factory()->set_supports_simulcast(false);
+  SetupCodec(/*active_streams=*/{true, false});
+  EXPECT_EQ(1u, helper_->factory()->encoders().size());
+
+  // Singlecast via layers deactivation. Highest layer is active.
+  ReSetUp();
+  helper_->factory()->set_supports_simulcast(false);
+  SetupCodec(/*active_streams=*/{false, true});
+  EXPECT_EQ(1u, helper_->factory()->encoders().size());
+}
+
+TEST_F(TestSimulcastEncoderAdapterFake,
+       RecreateEncoderIfPreferTemporalSupportIsEnabled) {
+  // Normally SEA reuses encoders. But, when TL-based SW fallback is enabled,
+  // the encoder which served the lowest stream should be recreated before it
+  // can be used to process an upper layer and vice-versa.
+  test::ScopedFieldTrials field_trials(
+      "WebRTC-Video-PreferTemporalSupportOnBaseLayer/Enabled/");
+  use_fallback_factory_ = true;
+  ReSetUp();
+
+  // Legacy singlecast
+  SetupCodec(/*active_streams=*/{});
+  ASSERT_EQ(1u, helper_->factory()->encoders().size());
+
+  // Singlecast, the lowest stream is active. Encoder should be reused.
+  MockVideoEncoder* prev_encoder = helper_->factory()->encoders()[0];
+  SetupCodec(/*active_streams=*/{true, false});
+  ASSERT_EQ(1u, helper_->factory()->encoders().size());
+  EXPECT_EQ(helper_->factory()->encoders()[0], prev_encoder);
+
+  // Singlecast, an upper stream is active. Encoder should be recreated.
+  EXPECT_CALL(*prev_encoder, Release()).Times(1);
+  SetupCodec(/*active_streams=*/{false, true});
+  ASSERT_EQ(1u, helper_->factory()->encoders().size());
+  EXPECT_NE(helper_->factory()->encoders()[0], prev_encoder);
+
+  // Singlecast, the lowest stream is active. Encoder should be recreated.
+  prev_encoder = helper_->factory()->encoders()[0];
+  EXPECT_CALL(*prev_encoder, Release()).Times(1);
+  SetupCodec(/*active_streams=*/{true, false});
+  ASSERT_EQ(1u, helper_->factory()->encoders().size());
+  EXPECT_NE(helper_->factory()->encoders()[0], prev_encoder);
+}
+
 }  // namespace test
 }  // namespace webrtc

media/engine/webrtc_video_engine.cc

@@ -2306,6 +2306,9 @@ webrtc::RTCError WebRtcVideoChannel::WebRtcVideoSendStream::SetRtpParameters(
   // TODO(bugs.webrtc.org/8807): The active field as well should not require
   // a full encoder reconfiguration, but it needs to update both the bitrate
   // allocator and the video bitrate allocator.
+  //
+  // Note that the simulcast encoder adapter relies on the fact that layers
+  // de/activation triggers encoder reinitialization.
   bool new_send_state = false;
   for (size_t i = 0; i < rtp_parameters_.encodings.size(); ++i) {
     bool new_active = IsLayerActive(new_parameters.encodings[i]);