WebRtcVideoChannel encoder fallback.

In this CL:
 - Added WEBRTC_VIDEO_CODEC_ENCODER_FAILURE return code that can
   be returned by the encoder wrapper in case of a broken encoder.
 - Added EncoderFailureCallback interface that can be called
   to request encoder fallback to be performed. Implemented by
   WebRtcVideoChannel and called from the VideoStreamEncoder.
 - Updated SelectSendVideoCodec to select all compatible codecs instead
   of just one.

Bug: webrtc:10795
Change-Id: I87a83fd02e48c40493c930471c06c3d0941031ab
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/140888
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Reviewed-by: Magnus Jedvert <magjed@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#28462}

api/video/video_stream_encoder_settings.h

@@ -17,6 +17,12 @@
 
 namespace webrtc {
 
+class EncoderFailureCallback {
+ public:
+  virtual ~EncoderFailureCallback() {}
+  virtual void OnEncoderFailure() = 0;
+};
+
 struct VideoStreamEncoderSettings {
   explicit VideoStreamEncoderSettings(
       const VideoEncoder::Capabilities& capabilities)
@@ -29,6 +35,9 @@ struct VideoStreamEncoderSettings {
   // Ownership stays with WebrtcVideoEngine (delegated from PeerConnection).
   VideoEncoderFactory* encoder_factory = nullptr;
 
+  // Notifies the WebRtcVideoChannel that the currently used encoder is broken.
+  EncoderFailureCallback* encoder_failure_callback = nullptr;
+
   // Ownership stays with WebrtcVideoEngine (delegated from PeerConnection).
   VideoBitrateAllocatorFactory* bitrate_allocator_factory = nullptr;
 

api/video_codecs/sdp_video_format.cc

@@ -9,6 +9,7 @@
  */
 
 #include "api/video_codecs/sdp_video_format.h"
+#include "rtc_base/strings/string_builder.h"
 
 namespace webrtc {
 
@@ -25,6 +26,16 @@ SdpVideoFormat& SdpVideoFormat::operator=(SdpVideoFormat&&) = default;
 
 SdpVideoFormat::~SdpVideoFormat() = default;
 
+std::string SdpVideoFormat::ToString() const {
+  rtc::StringBuilder builder;
+  builder << "Codec name: " << name << ", parameters: {";
+  for (const auto& kv : parameters)
+    builder << " " << kv.first << "=" << kv.second;
+  builder << " }";
+
+  return builder.str();
+}
+
 bool operator==(const SdpVideoFormat& a, const SdpVideoFormat& b) {
   return a.name == b.name && a.parameters == b.parameters;
 }

api/video_codecs/sdp_video_format.h

@@ -32,6 +32,8 @@ struct RTC_EXPORT SdpVideoFormat {
 
   ~SdpVideoFormat();
 
+  std::string ToString() const;
+
   friend RTC_EXPORT bool operator==(const SdpVideoFormat& a,
                                     const SdpVideoFormat& b);
   friend RTC_EXPORT bool operator!=(const SdpVideoFormat& a,

media/engine/webrtc_video_engine.cc

@@ -543,6 +543,7 @@ WebRtcVideoChannel::WebRtcVideoChannel(
     webrtc::VideoDecoderFactory* decoder_factory,
     webrtc::VideoBitrateAllocatorFactory* bitrate_allocator_factory)
     : VideoMediaChannel(config),
+      worker_thread_(rtc::Thread::Current()),
       call_(call),
       unsignalled_ssrc_handler_(&default_unsignalled_ssrc_handler_),
       video_config_(config.video),
@@ -575,23 +576,42 @@ WebRtcVideoChannel::~WebRtcVideoChannel() {
     delete kv.second;
 }
 
-absl::optional<WebRtcVideoChannel::VideoCodecSettings>
-WebRtcVideoChannel::SelectSendVideoCodec(
+std::vector<WebRtcVideoChannel::VideoCodecSettings>
+WebRtcVideoChannel::SelectSendVideoCodecs(
     const std::vector<VideoCodecSettings>& remote_mapped_codecs) const {
-  const std::vector<VideoCodec> local_supported_codecs =
-      AssignPayloadTypesAndDefaultCodecs(encoder_factory_);
-  // Select the first remote codec that is supported locally.
-  for (const VideoCodecSettings& remote_mapped_codec : remote_mapped_codecs) {
-    // For H264, we will limit the encode level to the remote offered level
-    // regardless if level asymmetry is allowed or not. This is strictly not
-    // following the spec in https://tools.ietf.org/html/rfc6184#section-8.2.2
-    // since we should limit the encode level to the lower of local and remote
-    // level when level asymmetry is not allowed.
-    if (FindMatchingCodec(local_supported_codecs, remote_mapped_codec.codec))
-      return remote_mapped_codec;
+  std::vector<webrtc::SdpVideoFormat> sdp_formats =
+      encoder_factory_->GetSupportedFormats();
+
+  // The returned vector holds the VideoCodecSettings in term of preference.
+  // They are orderd by receive codec preference first and local implementation
+  // preference second.
+  std::vector<VideoCodecSettings> encoders;
+  for (const VideoCodecSettings& remote_codec : remote_mapped_codecs) {
+    for (auto format_it = sdp_formats.begin();
+         format_it != sdp_formats.end();) {
+      // For H264, we will limit the encode level to the remote offered level
+      // regardless if level asymmetry is allowed or not. This is strictly not
+      // following the spec in https://tools.ietf.org/html/rfc6184#section-8.2.2
+      // since we should limit the encode level to the lower of local and remote
+      // level when level asymmetry is not allowed.
+      if (IsSameCodec(format_it->name, format_it->parameters,
+                      remote_codec.codec.name, remote_codec.codec.params)) {
+        encoders.push_back(remote_codec);
+
+        // To allow the VideoEncoderFactory to keep information about which
+        // implementation to instantitate when CreateEncoder is called the two
+        // parmeter sets are merged.
+        encoders.back().codec.params.insert(format_it->parameters.begin(),
+                                            format_it->parameters.end());
+
+        format_it = sdp_formats.erase(format_it);
+      } else {
+        ++format_it;
+      }
+    }
   }
-  // No remote codec was supported.
-  return absl::nullopt;
+
+  return encoders;
 }
 
 bool WebRtcVideoChannel::NonFlexfecReceiveCodecsHaveChanged(
@@ -627,27 +647,27 @@ bool WebRtcVideoChannel::GetChangedSendParameters(
     return false;
   }
 
-  // Select one of the remote codecs that will be used as send codec.
-  absl::optional<VideoCodecSettings> selected_send_codec =
-      SelectSendVideoCodec(MapCodecs(params.codecs));
+  std::vector<VideoCodecSettings> negotiated_codecs =
+      SelectSendVideoCodecs(MapCodecs(params.codecs));
 
-  if (!selected_send_codec) {
+  if (negotiated_codecs.empty()) {
     RTC_LOG(LS_ERROR) << "No video codecs supported.";
     return false;
   }
 
   // Never enable sending FlexFEC, unless we are in the experiment.
   if (!IsFlexfecFieldTrialEnabled()) {
-    if (selected_send_codec->flexfec_payload_type != -1) {
-      RTC_LOG(LS_INFO)
-          << "Remote supports flexfec-03, but we will not send since "
-          << "WebRTC-FlexFEC-03 field trial is not enabled.";
-    }
-    selected_send_codec->flexfec_payload_type = -1;
+    RTC_LOG(LS_INFO) << "WebRTC-FlexFEC-03 field trial is not enabled.";
+    for (VideoCodecSettings& codec : negotiated_codecs)
+      codec.flexfec_payload_type = -1;
   }
 
-  if (!send_codec_ || *selected_send_codec != *send_codec_)
-    changed_params->codec = selected_send_codec;
+  if (negotiated_codecs_ != negotiated_codecs) {
+    if (send_codec_ != negotiated_codecs.front()) {
+      changed_params->send_codec = negotiated_codecs.front();
+    }
+    changed_params->negotiated_codecs = std::move(negotiated_codecs);
+  }
 
   // Handle RTP header extensions.
   if (params.extmap_allow_mixed != ExtmapAllowMixed()) {
@@ -698,12 +718,44 @@ bool WebRtcVideoChannel::SetSendParameters(const VideoSendParameters& params) {
     return false;
   }
 
-  if (changed_params.codec) {
-    const VideoCodecSettings& codec_settings = *changed_params.codec;
-    send_codec_ = codec_settings;
-    RTC_LOG(LS_INFO) << "Using codec: " << codec_settings.codec.ToString();
+  if (changed_params.negotiated_codecs) {
+    for (const auto& send_codec : *changed_params.negotiated_codecs)
+      RTC_LOG(LS_INFO) << "Negotiated codec: " << send_codec.codec.ToString();
   }
 
+  send_params_ = params;
+  return ApplyChangedParams(changed_params);
+}
+
+void WebRtcVideoChannel::OnEncoderFailure() {
+  invoker_.AsyncInvoke<void>(
+      RTC_FROM_HERE, worker_thread_, [this] {
+        RTC_DCHECK_RUN_ON(&thread_checker_);
+        if (negotiated_codecs_.size() <= 1) {
+          RTC_LOG(LS_WARNING)
+              << "Encoder failed but no fallback codec is available";
+          return;
+        }
+
+        ChangedSendParameters params;
+        params.negotiated_codecs = negotiated_codecs_;
+        params.negotiated_codecs->erase(params.negotiated_codecs->begin());
+        params.send_codec = params.negotiated_codecs->front();
+        ApplyChangedParams(params);
+      });
+}
+
+bool WebRtcVideoChannel::ApplyChangedParams(
+    const ChangedSendParameters& changed_params) {
+  RTC_DCHECK_RUN_ON(&thread_checker_);
+  if (changed_params.negotiated_codecs)
+    negotiated_codecs_ = *changed_params.negotiated_codecs;
+
+  if (changed_params.send_codec)
+    send_codec_ = changed_params.send_codec;
+
+  RTC_DCHECK(send_codec_);
+
   if (changed_params.extmap_allow_mixed) {
     SetExtmapAllowMixed(*changed_params.extmap_allow_mixed);
   }
@@ -711,8 +763,8 @@ bool WebRtcVideoChannel::SetSendParameters(const VideoSendParameters& params) {
     send_rtp_extensions_ = changed_params.rtp_header_extensions;
   }
 
-  if (changed_params.codec || changed_params.max_bandwidth_bps) {
-    if (params.max_bandwidth_bps == -1) {
+  if (changed_params.send_codec || changed_params.max_bandwidth_bps) {
+    if (send_params_.max_bandwidth_bps == -1) {
       // Unset the global max bitrate (max_bitrate_bps) if max_bandwidth_bps is
       // -1, which corresponds to no "b=AS" attribute in SDP. Note that the
       // global max bitrate may be set below in GetBitrateConfigForCodec, from
@@ -721,17 +773,19 @@ bool WebRtcVideoChannel::SetSendParameters(const VideoSendParameters& params) {
       // probably not affect global call max bitrate).
       bitrate_config_.max_bitrate_bps = -1;
     }
+
     if (send_codec_) {
       // TODO(holmer): Changing the codec parameters shouldn't necessarily mean
       // that we change the min/max of bandwidth estimation. Reevaluate this.
       bitrate_config_ = GetBitrateConfigForCodec(send_codec_->codec);
-      if (!changed_params.codec) {
+      if (!changed_params.send_codec) {
         // If the codec isn't changing, set the start bitrate to -1 which means
         // "unchanged" so that BWE isn't affected.
         bitrate_config_.start_bitrate_bps = -1;
       }
     }
-    if (params.max_bandwidth_bps >= 0) {
+
+    if (send_params_.max_bandwidth_bps >= 0) {
       // Note that max_bandwidth_bps intentionally takes priority over the
       // bitrate config for the codec. This allows FEC to be applied above the
       // codec target bitrate.
@@ -739,8 +793,9 @@ bool WebRtcVideoChannel::SetSendParameters(const VideoSendParameters& params) {
       // WebRtcVideoChannel (in which case we're good), or per sender (SSRC),
       // in which case this should not set a BitrateConstraints but rather
       // reconfigure all senders.
-      bitrate_config_.max_bitrate_bps =
-          params.max_bandwidth_bps == 0 ? -1 : params.max_bandwidth_bps;
+      bitrate_config_.max_bitrate_bps = send_params_.max_bandwidth_bps == 0
+                                            ? -1
+                                            : send_params_.max_bandwidth_bps;
     }
 
     if (media_transport()) {
@@ -767,7 +822,7 @@ bool WebRtcVideoChannel::SetSendParameters(const VideoSendParameters& params) {
     for (auto& kv : send_streams_) {
       kv.second->SetSendParameters(changed_params);
     }
-    if (changed_params.codec || changed_params.rtcp_mode) {
+    if (changed_params.send_codec || changed_params.rtcp_mode) {
       // Update receive feedback parameters from new codec or RTCP mode.
       RTC_LOG(LS_INFO)
           << "SetFeedbackOptions on all the receive streams because the send "
@@ -777,11 +832,10 @@ bool WebRtcVideoChannel::SetSendParameters(const VideoSendParameters& params) {
         kv.second->SetFeedbackParameters(
             HasLntf(send_codec_->codec), HasNack(send_codec_->codec),
             HasRemb(send_codec_->codec), HasTransportCc(send_codec_->codec),
-            params.rtcp.reduced_size ? webrtc::RtcpMode::kReducedSize
-                                     : webrtc::RtcpMode::kCompound);
+            send_params_.rtcp.reduced_size ? webrtc::RtcpMode::kReducedSize
+                                           : webrtc::RtcpMode::kCompound);
       }
     }
-  send_params_ = params;
   return true;
 }
 
@@ -1107,6 +1161,7 @@ bool WebRtcVideoChannel::AddSendStream(const StreamParams& sp) {
   config.encoder_settings.encoder_factory = encoder_factory_;
   config.encoder_settings.bitrate_allocator_factory =
       bitrate_allocator_factory_;
+  config.encoder_settings.encoder_failure_callback = this;
   config.crypto_options = crypto_options_;
   config.rtp.extmap_allow_mixed = ExtmapAllowMixed();
   config.rtcp_report_interval_ms = video_config_.rtcp_report_interval_ms;
@@ -1949,8 +2004,8 @@ void WebRtcVideoChannel::WebRtcVideoSendStream::SetSendParameters(
   }
 
   // Set codecs and options.
-  if (params.codec) {
-    SetCodec(*params.codec);
+  if (params.send_codec) {
+    SetCodec(*params.send_codec);
     recreate_stream = false;  // SetCodec has already recreated the stream.
   } else if (params.conference_mode && parameters_.codec_settings) {
     SetCodec(*parameters_.codec_settings);

media/engine/webrtc_video_engine.h

@@ -106,7 +106,9 @@ class WebRtcVideoEngine : public VideoEngineInterface {
       bitrate_allocator_factory_;
 };
 
-class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport {
+class WebRtcVideoChannel : public VideoMediaChannel,
+                           public webrtc::Transport,
+                           public webrtc::EncoderFailureCallback {
  public:
   WebRtcVideoChannel(
       webrtc::Call* call,
@@ -205,6 +207,9 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport {
   // This method does nothing unless unknown_ssrc_packet_buffer_ is configured.
   void BackfillBufferedPackets(rtc::ArrayView<const uint32_t> ssrcs);
 
+  // Implements webrtc::EncoderFailureCallback.
+  void OnEncoderFailure() override;
+
  private:
   class WebRtcVideoReceiveStream;
   struct VideoCodecSettings {
@@ -228,7 +233,8 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport {
 
   struct ChangedSendParameters {
     // These optionals are unset if not changed.
-    absl::optional<VideoCodecSettings> codec;
+    absl::optional<VideoCodecSettings> send_codec;
+    absl::optional<std::vector<VideoCodecSettings>> negotiated_codecs;
     absl::optional<std::vector<webrtc::RtpExtension>> rtp_header_extensions;
     absl::optional<std::string> mid;
     absl::optional<bool> extmap_allow_mixed;
@@ -250,6 +256,7 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport {
   bool GetChangedSendParameters(const VideoSendParameters& params,
                                 ChangedSendParameters* changed_params) const
       RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
+  bool ApplyChangedParams(const ChangedSendParameters& changed_params);
   bool GetChangedRecvParameters(const VideoRecvParameters& params,
                                 ChangedRecvParameters* changed_params) const
       RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
@@ -474,10 +481,8 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport {
 
   static std::vector<VideoCodecSettings> MapCodecs(
       const std::vector<VideoCodec>& codecs);
-  // Select what video codec will be used for sending, i.e. what codec is used
-  // for local encoding, based on supported remote codecs. The first remote
-  // codec that is supported locally will be selected.
-  absl::optional<VideoCodecSettings> SelectSendVideoCodec(
+  // Get all codecs that are compatible with the receiver.
+  std::vector<VideoCodecSettings> SelectSendVideoCodecs(
       const std::vector<VideoCodecSettings>& remote_mapped_codecs) const
       RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
 
@@ -495,6 +500,7 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport {
   void FillSendAndReceiveCodecStats(VideoMediaInfo* video_media_info)
       RTC_EXCLUSIVE_LOCKS_REQUIRED(thread_checker_);
 
+  rtc::Thread* worker_thread_;
   rtc::ThreadChecker thread_checker_;
 
   uint32_t rtcp_receiver_report_ssrc_ RTC_GUARDED_BY(thread_checker_);
@@ -521,6 +527,9 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport {
 
   absl::optional<VideoCodecSettings> send_codec_
       RTC_GUARDED_BY(thread_checker_);
+  std::vector<VideoCodecSettings> negotiated_codecs_
+      RTC_GUARDED_BY(thread_checker_);
+
   absl::optional<std::vector<webrtc::RtpExtension>> send_rtp_extensions_
       RTC_GUARDED_BY(thread_checker_);
 
@@ -556,6 +565,10 @@ class WebRtcVideoChannel : public VideoMediaChannel, public webrtc::Transport {
   // Buffer for unhandled packets.
   std::unique_ptr<UnhandledPacketsBuffer> unknown_ssrc_packet_buffer_
       RTC_GUARDED_BY(thread_checker_);
+
+  // In order for the |invoker_| to protect other members from being destructed
+  // as they are used in asynchronous tasks it has to be destructed first.
+  rtc::AsyncInvoker invoker_;
 };
 
 class EncoderStreamFactory

media/engine/webrtc_video_engine_unittest.cc

@@ -2082,6 +2082,30 @@ TEST_F(WebRtcVideoChannelBaseTest, TwoStreamsSendAndReceive) {
   TwoStreamsSendAndReceive(codec);
 }
 
+TEST_F(WebRtcVideoChannelBaseTest, OnEncoderFailure) {
+  cricket::VideoSendParameters parameters;
+  parameters.codecs.push_back(GetEngineCodec("VP9"));
+  parameters.codecs.push_back(GetEngineCodec("VP8"));
+  EXPECT_TRUE(channel_->SetSendParameters(parameters));
+
+  VideoCodec codec;
+  ASSERT_TRUE(channel_->GetSendCodec(&codec));
+  EXPECT_EQ("VP9", codec.name);
+
+  // OnEncoderFailure will post a task to the worker thread (which is also
+  // the current thread), hence the ProcessMessages call.
+  channel_->OnEncoderFailure();
+  rtc::Thread::Current()->ProcessMessages(30);
+  ASSERT_TRUE(channel_->GetSendCodec(&codec));
+  EXPECT_EQ("VP8", codec.name);
+
+  // No other codec to fall back to, keep using VP8.
+  channel_->OnEncoderFailure();
+  rtc::Thread::Current()->ProcessMessages(30);
+  ASSERT_TRUE(channel_->GetSendCodec(&codec));
+  EXPECT_EQ("VP8", codec.name);
+}
+
 class WebRtcVideoChannelTest : public WebRtcVideoEngineTest {
  public:
   WebRtcVideoChannelTest() : WebRtcVideoChannelTest("") {}

modules/video_coding/include/video_error_codes.h

@@ -25,5 +25,6 @@
 #define WEBRTC_VIDEO_CODEC_FALLBACK_SOFTWARE -13
 #define WEBRTC_VIDEO_CODEC_TARGET_BITRATE_OVERSHOOT -14
 #define WEBRTC_VIDEO_CODEC_ERR_SIMULCAST_PARAMETERS_NOT_SUPPORTED -15
+#define WEBRTC_VIDEO_CODEC_ENCODER_FAILURE -16
 
 #endif  // MODULES_VIDEO_CODING_INCLUDE_VIDEO_ERROR_CODES_H_

video/video_stream_encoder.cc

@@ -491,6 +491,7 @@ VideoStreamEncoder::VideoStreamEncoder(
       max_data_payload_length_(0),
       encoder_paused_and_dropped_frame_(false),
       was_encode_called_since_last_initialization_(false),
+      encoder_failed_(false),
       clock_(clock),
       degradation_preference_(DegradationPreference::DISABLED),
       posted_frames_waiting_for_encode_(0),
@@ -1262,6 +1263,13 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame,
 void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame,
                                           int64_t time_when_posted_us) {
   RTC_DCHECK_RUN_ON(&encoder_queue_);
+
+  // If the encoder fail we can't continue to encode frames. When this happens
+  // the WebrtcVideoSender is notified and the whole VideoSendStream is
+  // recreated.
+  if (encoder_failed_)
+    return;
+
   TraceFrameDropEnd();
 
   VideoFrame out_frame(video_frame);
@@ -1387,8 +1395,21 @@ void VideoStreamEncoder::EncodeVideoFrame(const VideoFrame& video_frame,
   was_encode_called_since_last_initialization_ = true;
 
   if (encode_status < 0) {
-    RTC_LOG(LS_ERROR) << "Failed to encode frame. Error code: "
-                      << encode_status;
+    if (encode_status == WEBRTC_VIDEO_CODEC_ENCODER_FAILURE) {
+      RTC_LOG(LS_ERROR) << "Encoder failed, failing encoder format: "
+                        << encoder_config_.video_format.ToString();
+      if (settings_.encoder_failure_callback) {
+        encoder_failed_ = true;
+        settings_.encoder_failure_callback->OnEncoderFailure();
+      } else {
+        RTC_LOG(LS_ERROR)
+            << "Encoder failed but no encoder fallback callback is registered";
+      }
+    } else {
+      RTC_LOG(LS_ERROR) << "Failed to encode frame. Error code: "
+                        << encode_status;
+    }
+
     return;
   }
 

video/video_stream_encoder.h

@@ -281,6 +281,7 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
   bool was_encode_called_since_last_initialization_
       RTC_GUARDED_BY(&encoder_queue_);
 
+  bool encoder_failed_ RTC_GUARDED_BY(&encoder_queue_);
   Clock* const clock_;
   // Counters used for deciding if the video resolution or framerate is
   // currently restricted, and if so, why, on a per degradation preference