Insert frame transformer between Encoded and Packetizer.

Add a new API in RTPSenderInterface, to be called from the browser side
to insert a frame transformer between the Encoded and the Packetizer.

The frame transformer is passed from RTPSenderInterface through the
library to be eventually set in RTPSenderVideo, where the frame
transformation will occur in the follow-up CL
https://webrtc-review.googlesource.com/c/src/+/169128.

Insertable Streams Web API explainer:
https://github.com/alvestrand/webrtc-media-streams/blob/master/explainer.md

Design doc for WebRTC library changes:
http://doc/1eiLkjNUkRy2FssCPLUp6eH08BZuXXoHfbbBP1ZN7EVk

Bug: webrtc:11380
Change-Id: I46cd0d8a798c2736c837e90cbf90d8901c7d27fb
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/169127
Commit-Queue: Marina Ciocea <marinaciocea@webrtc.org>
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Reviewed-by: Karl Wiberg <kwiberg@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30642}

api/rtp_sender_interface.cc

@@ -30,4 +30,7 @@ rtc::scoped_refptr<DtlsTransportInterface> RtpSenderInterface::dtls_transport()
   return nullptr;
 }
 
+void RtpSenderInterface::SetEncoderToPacketizerFrameTransformer(
+    rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) {}
+
 }  // namespace webrtc

api/rtp_sender_interface.h

@@ -20,6 +20,7 @@
 #include "api/crypto/frame_encryptor_interface.h"
 #include "api/dtls_transport_interface.h"
 #include "api/dtmf_sender_interface.h"
+#include "api/frame_transformer_interface.h"
 #include "api/media_stream_interface.h"
 #include "api/media_types.h"
 #include "api/proxy.h"
@@ -93,6 +94,9 @@ class RTC_EXPORT RtpSenderInterface : public rtc::RefCountInterface {
   // user. This can be used to update the state of the object.
   virtual rtc::scoped_refptr<FrameEncryptorInterface> GetFrameEncryptor() const;
 
+  virtual void SetEncoderToPacketizerFrameTransformer(
+      rtc::scoped_refptr<FrameTransformerInterface> frame_transformer);
+
  protected:
   ~RtpSenderInterface() override = default;
 };
@@ -119,6 +123,9 @@ PROXY_METHOD1(void,
 PROXY_CONSTMETHOD0(rtc::scoped_refptr<FrameEncryptorInterface>,
                    GetFrameEncryptor)
 PROXY_METHOD1(void, SetStreams, const std::vector<std::string>&)
+PROXY_METHOD1(void,
+              SetEncoderToPacketizerFrameTransformer,
+              rtc::scoped_refptr<FrameTransformerInterface>)
 END_PROXY_MAP()
 
 }  // namespace webrtc

call/BUILD.gn

@@ -89,6 +89,7 @@ rtc_library("rtp_interfaces") {
   deps = [
     "../api:array_view",
     "../api:fec_controller_api",
+    "../api:frame_transformer_interface",
     "../api:rtp_headers",
     "../api:rtp_parameters",
     "../api/crypto:options",
@@ -293,6 +294,7 @@ rtc_library("video_stream_api") {
   ]
   deps = [
     ":rtp_interfaces",
+    "../api:frame_transformer_interface",
     "../api:rtp_headers",
     "../api:rtp_parameters",
     "../api:transport_api",
@@ -501,6 +503,7 @@ if (rtc_include_tests) {
     ]
     deps = [
       ":rtp_interfaces",
+      "../api:frame_transformer_interface",
       "../api:libjingle_peerconnection_api",
       "../api/crypto:frame_encryptor_interface",
       "../api/crypto:options",

call/rtp_transport_controller_send.cc

@@ -139,7 +139,8 @@ RtpVideoSenderInterface* RtpTransportControllerSend::CreateRtpVideoSender(
     const RtpSenderObservers& observers,
     RtcEventLog* event_log,
     std::unique_ptr<FecController> fec_controller,
-    const RtpSenderFrameEncryptionConfig& frame_encryption_config) {
+    const RtpSenderFrameEncryptionConfig& frame_encryption_config,
+    rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) {
   video_rtp_senders_.push_back(std::make_unique<RtpVideoSender>(
       clock_, suspended_ssrcs, states, rtp_config, rtcp_report_interval_ms,
       send_transport, observers,
@@ -147,7 +148,7 @@ RtpVideoSenderInterface* RtpTransportControllerSend::CreateRtpVideoSender(
       // the parts of RtpTransportControllerSendInterface that are really used.
       this, event_log, &retransmission_rate_limiter_, std::move(fec_controller),
       frame_encryption_config.frame_encryptor,
-      frame_encryption_config.crypto_options));
+      frame_encryption_config.crypto_options, std::move(frame_transformer)));
   return video_rtp_senders_.back().get();
 }
 

call/rtp_transport_controller_send.h

@@ -71,7 +71,8 @@ class RtpTransportControllerSend final
       const RtpSenderObservers& observers,
       RtcEventLog* event_log,
       std::unique_ptr<FecController> fec_controller,
-      const RtpSenderFrameEncryptionConfig& frame_encryption_config) override;
+      const RtpSenderFrameEncryptionConfig& frame_encryption_config,
+      rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) override;
   void DestroyRtpVideoSender(
       RtpVideoSenderInterface* rtp_video_sender) override;
 

call/rtp_transport_controller_send_interface.h

@@ -21,6 +21,7 @@
 #include "absl/types/optional.h"
 #include "api/crypto/crypto_options.h"
 #include "api/fec_controller.h"
+#include "api/frame_transformer_interface.h"
 #include "api/rtc_event_log/rtc_event_log.h"
 #include "api/transport/bitrate_settings.h"
 #include "api/units/timestamp.h"
@@ -110,7 +111,8 @@ class RtpTransportControllerSendInterface {
       const RtpSenderObservers& observers,
       RtcEventLog* event_log,
       std::unique_ptr<FecController> fec_controller,
-      const RtpSenderFrameEncryptionConfig& frame_encryption_config) = 0;
+      const RtpSenderFrameEncryptionConfig& frame_encryption_config,
+      rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) = 0;
   virtual void DestroyRtpVideoSender(
       RtpVideoSenderInterface* rtp_video_sender) = 0;
 

call/rtp_video_sender.cc

@@ -126,7 +126,8 @@ std::vector<RtpStreamSender> CreateRtpStreamSenders(
     RateLimiter* retransmission_rate_limiter,
     OverheadObserver* overhead_observer,
     FrameEncryptorInterface* frame_encryptor,
-    const CryptoOptions& crypto_options) {
+    const CryptoOptions& crypto_options,
+    rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) {
   RTC_DCHECK_GT(rtp_config.ssrcs.size(), 0);
 
   RtpRtcp::Configuration configuration;
@@ -206,6 +207,7 @@ std::vector<RtpStreamSender> CreateRtpStreamSenders(
         !should_disable_red_and_ulpfec) {
       video_config.ulpfec_payload_type = rtp_config.ulpfec.ulpfec_payload_type;
     }
+    video_config.frame_transformer = std::move(frame_transformer);
     auto sender_video = std::make_unique<RTPSenderVideo>(video_config);
     rtp_streams.emplace_back(std::move(rtp_rtcp), std::move(sender_video));
   }
@@ -291,7 +293,8 @@ RtpVideoSender::RtpVideoSender(
     RateLimiter* retransmission_limiter,
     std::unique_ptr<FecController> fec_controller,
     FrameEncryptorInterface* frame_encryptor,
-    const CryptoOptions& crypto_options)
+    const CryptoOptions& crypto_options,
+    rtc::scoped_refptr<FrameTransformerInterface> frame_transformer)
     : send_side_bwe_with_overhead_(
           webrtc::field_trial::IsEnabled("WebRTC-SendSideBwe-WithOverhead")),
       account_for_packetization_overhead_(!webrtc::field_trial::IsDisabled(
@@ -318,7 +321,8 @@ RtpVideoSender::RtpVideoSender(
                                           retransmission_limiter,
                                           this,
                                           frame_encryptor,
-                                          crypto_options)),
+                                          crypto_options,
+                                          std::move(frame_transformer))),
       rtp_config_(rtp_config),
       codec_type_(GetVideoCodecType(rtp_config)),
       transport_(transport),

call/rtp_video_sender.h

@@ -85,7 +85,8 @@ class RtpVideoSender : public RtpVideoSenderInterface,
       RateLimiter* retransmission_limiter,  // move inside RtpTransport
       std::unique_ptr<FecController> fec_controller,
       FrameEncryptorInterface* frame_encryptor,
-      const CryptoOptions& crypto_options);  // move inside RtpTransport
+      const CryptoOptions& crypto_options,  // move inside RtpTransport
+      rtc::scoped_refptr<FrameTransformerInterface> frame_transformer);
   ~RtpVideoSender() override;
 
   // RegisterProcessThread register |module_process_thread| with those objects

call/rtp_video_sender_unittest.cc

@@ -151,7 +151,7 @@ class RtpVideoSenderTestFixture {
                         &send_delay_stats_),
         &transport_controller_, &event_log_, &retransmission_rate_limiter_,
         std::make_unique<FecControllerDefault>(time_controller_.GetClock()),
-        nullptr, CryptoOptions{});
+        nullptr, CryptoOptions{}, nullptr);
   }
   RtpVideoSenderTestFixture(
       const std::vector<uint32_t>& ssrcs,

call/test/mock_rtp_transport_controller_send.h

@@ -18,6 +18,7 @@
 
 #include "api/crypto/crypto_options.h"
 #include "api/crypto/frame_encryptor_interface.h"
+#include "api/frame_transformer_interface.h"
 #include "api/transport/bitrate_settings.h"
 #include "call/rtp_transport_controller_send_interface.h"
 #include "modules/pacing/packet_router.h"
@@ -31,7 +32,7 @@ namespace webrtc {
 class MockRtpTransportControllerSend
     : public RtpTransportControllerSendInterface {
  public:
-  MOCK_METHOD9(
+  MOCK_METHOD10(
       CreateRtpVideoSender,
       RtpVideoSenderInterface*(std::map<uint32_t, RtpState>,
                                const std::map<uint32_t, RtpPayloadState>&,
@@ -41,7 +42,8 @@ class MockRtpTransportControllerSend
                                const RtpSenderObservers&,
                                RtcEventLog*,
                                std::unique_ptr<FecController>,
-                               const RtpSenderFrameEncryptionConfig&));
+                               const RtpSenderFrameEncryptionConfig&,
+                               rtc::scoped_refptr<FrameTransformerInterface>));
   MOCK_METHOD1(DestroyRtpVideoSender, void(RtpVideoSenderInterface*));
   MOCK_METHOD0(GetWorkerQueue, rtc::TaskQueue*());
   MOCK_METHOD0(packet_router, PacketRouter*());

call/video_send_stream.h

@@ -20,6 +20,7 @@
 #include "absl/types/optional.h"
 #include "api/call/transport.h"
 #include "api/crypto/crypto_options.h"
+#include "api/frame_transformer_interface.h"
 #include "api/rtp_parameters.h"
 #include "api/video/video_content_type.h"
 #include "api/video/video_frame.h"
@@ -163,6 +164,8 @@ class VideoSendStream {
     // Per PeerConnection cryptography options.
     CryptoOptions crypto_options;
 
+    rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer;
+
    private:
     // Access to the copy constructor is private to force use of the Copy()
     // method for those exceptional cases where we do use it.

media/BUILD.gn

@@ -86,6 +86,7 @@ rtc_library("rtc_media_base") {
     "..:webrtc_common",
     "../api:array_view",
     "../api:audio_options_api",
+    "../api:frame_transformer_interface",
     "../api:media_stream_interface",
     "../api:rtc_error",
     "../api:rtp_parameters",

media/base/media_channel.cc

@@ -49,6 +49,10 @@ void MediaChannel::SetFrameDecryptor(
 
 void MediaChannel::SetVideoCodecSwitchingEnabled(bool enabled) {}
 
+void MediaChannel::SetEncoderToPacketizerFrameTransformer(
+    uint32_t ssrc,
+    rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {}
+
 MediaSenderInfo::MediaSenderInfo() = default;
 MediaSenderInfo::~MediaSenderInfo() = default;
 

media/base/media_channel.h

@@ -22,6 +22,7 @@
 #include "api/audio_options.h"
 #include "api/crypto/frame_decryptor_interface.h"
 #include "api/crypto/frame_encryptor_interface.h"
+#include "api/frame_transformer_interface.h"
 #include "api/rtc_error.h"
 #include "api/rtp_parameters.h"
 #include "api/transport/media/media_transport_config.h"
@@ -287,6 +288,10 @@ class MediaChannel : public sigslot::has_slots<> {
       uint32_t ssrc,
       const webrtc::RtpParameters& parameters) = 0;
 
+  virtual void SetEncoderToPacketizerFrameTransformer(
+      uint32_t ssrc,
+      rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer);
+
  protected:
   bool DscpEnabled() const { return enable_dscp_; }
 

media/engine/webrtc_video_engine.cc

@@ -2464,6 +2464,14 @@ void WebRtcVideoChannel::WebRtcVideoSendStream::FillBitrateInfo(
   bwe_info->actual_enc_bitrate += stats.media_bitrate_bps;
 }
 
+void WebRtcVideoChannel::WebRtcVideoSendStream::
+    SetEncoderToPacketizerFrameTransformer(
+        rtc::scoped_refptr<webrtc::FrameTransformerInterface>
+            frame_transformer) {
+  RTC_DCHECK_RUN_ON(&thread_checker_);
+  parameters_.config.frame_transformer = std::move(frame_transformer);
+}
+
 void WebRtcVideoChannel::WebRtcVideoSendStream::RecreateWebRtcStream() {
   RTC_DCHECK_RUN_ON(&thread_checker_);
   if (stream_ != NULL) {
@@ -3133,6 +3141,17 @@ void WebRtcVideoChannel::GenerateKeyFrame(uint32_t ssrc) {
   }
 }
 
+void WebRtcVideoChannel::SetEncoderToPacketizerFrameTransformer(
+    uint32_t ssrc,
+    rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
+  RTC_DCHECK_RUN_ON(&thread_checker_);
+  auto matching_stream = send_streams_.find(ssrc);
+  if (matching_stream != send_streams_.end()) {
+    matching_stream->second->SetEncoderToPacketizerFrameTransformer(
+        std::move(frame_transformer));
+  }
+}
+
 // TODO(bugs.webrtc.org/8785): Consider removing max_qp as member of
 // EncoderStreamFactory and instead set this value individually for each stream
 // in the VideoEncoderConfig.simulcast_layers.

media/engine/webrtc_video_engine.h

@@ -224,6 +224,11 @@ class WebRtcVideoChannel : public VideoMediaChannel,
   void ClearRecordableEncodedFrameCallback(uint32_t ssrc) override;
   void GenerateKeyFrame(uint32_t ssrc) override;
 
+  void SetEncoderToPacketizerFrameTransformer(
+      uint32_t ssrc,
+      rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer)
+      override;
+
  private:
   class WebRtcVideoReceiveStream;
 
@@ -336,6 +341,10 @@ class WebRtcVideoChannel : public VideoMediaChannel,
     VideoSenderInfo GetVideoSenderInfo(bool log_stats);
     void FillBitrateInfo(BandwidthEstimationInfo* bwe_info);
 
+    void SetEncoderToPacketizerFrameTransformer(
+        rtc::scoped_refptr<webrtc::FrameTransformerInterface>
+            frame_transformer);
+
    private:
     // Parameters needed to reconstruct the underlying stream.
     // webrtc::VideoSendStream doesn't support setting a lot of options on the

modules/rtp_rtcp/BUILD.gn

@@ -239,6 +239,7 @@ rtc_library("rtp_rtcp") {
     "..:module_fec_api",
     "../..:webrtc_common",
     "../../api:array_view",
+    "../../api:frame_transformer_interface",
     "../../api:function_view",
     "../../api:libjingle_peerconnection_api",
     "../../api:rtp_headers",

modules/rtp_rtcp/include/rtp_rtcp.h

@@ -19,6 +19,8 @@
 
 #include "absl/strings/string_view.h"
 #include "absl/types/optional.h"
+#include "api/frame_transformer_interface.h"
+#include "api/scoped_refptr.h"
 #include "api/transport/webrtc_key_value_config.h"
 #include "api/video/video_bitrate_allocation.h"
 #include "modules/include/module.h"
@@ -109,6 +111,8 @@ class RtpRtcp : public Module, public RtcpFeedbackSenderInterface {
     // Update network2 instead of pacer_exit field of video timing extension.
     bool populate_network2_timestamp = false;
 
+    rtc::scoped_refptr<FrameTransformerInterface> frame_transformer;
+
     // E2EE Custom Video Frame Encryption
     FrameEncryptorInterface* frame_encryptor = nullptr;
     // Require all outgoing frames to be encrypted with a FrameEncryptor.

modules/rtp_rtcp/source/rtp_sender_video.cc

@@ -52,9 +52,9 @@ constexpr int64_t kMaxUnretransmittableFrameIntervalMs = 33 * 4;
 // result recovered packets will be corrupt unless we also remove transport
 // sequence number during FEC calculation.
 //
-// TODO(sukhanov): We need to find find better way to implement FEC with
+// TODO(sukhanov): We need to find a better way to implement FEC with datagram
-// datagram transport, probably moving FEC to datagram integration layter. We
+// transport, probably moving FEC to datagram integration layter. We should
-// should also remove special field trial once we switch datagram path from
+// also remove special field trial once we switch datagram path from
 // RTCConfiguration flags to field trial and use the same field trial for FEC
 // workaround.
 const char kExcludeTransportSequenceNumberFromFecFieldTrial[] =
@@ -275,7 +275,8 @@ RTPSenderVideo::RTPSenderVideo(const Config& config)
           config.field_trials
               ->Lookup(kExcludeTransportSequenceNumberFromFecFieldTrial)
               .find("Enabled") == 0),
-      absolute_capture_time_sender_(config.clock) {}
+      absolute_capture_time_sender_(config.clock),
+      frame_transformer_(config.frame_transformer) {}
 
 RTPSenderVideo::~RTPSenderVideo() {}
 

modules/rtp_rtcp/source/rtp_sender_video.h

@@ -18,6 +18,8 @@
 #include "absl/strings/string_view.h"
 #include "absl/types/optional.h"
 #include "api/array_view.h"
+#include "api/frame_transformer_interface.h"
+#include "api/scoped_refptr.h"
 #include "api/transport/rtp/dependency_descriptor.h"
 #include "api/video/video_codec_type.h"
 #include "api/video/video_frame_type.h"
@@ -74,6 +76,7 @@ class RTPSenderVideo {
     absl::optional<int> red_payload_type;
     absl::optional<int> ulpfec_payload_type;
     const WebRtcKeyValueConfig* field_trials = nullptr;
+    rtc::scoped_refptr<FrameTransformerInterface> frame_transformer;
   };
 
   explicit RTPSenderVideo(const Config& config);
@@ -222,6 +225,8 @@ class RTPSenderVideo {
   const bool exclude_transport_sequence_number_from_fec_experiment_;
 
   AbsoluteCaptureTimeSender absolute_capture_time_sender_;
+
+  const rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_;
 };
 
 }  // namespace webrtc

pc/BUILD.gn

@@ -231,6 +231,7 @@ rtc_library("peerconnection") {
     "../api:audio_options_api",
     "../api:call_api",
     "../api:fec_controller_api",
+    "../api:frame_transformer_interface",
     "../api:ice_transport_factory",
     "../api:libjingle_peerconnection_api",
     "../api:media_stream_interface",
@@ -624,7 +625,8 @@ if (rtc_include_tests) {
       "../api/video_codecs:video_codecs_api",
       "../call:call_interfaces",
       "../media:rtc_audio_video",
-      "../media:rtc_data",  # TODO(phoglund): AFAIK only used for one sctp constant.
+      "../media:rtc_data",  # TODO(phoglund): AFAIK only used for one sctp
+                            # constant.
       "../media:rtc_media_base",
       "../media:rtc_media_tests_utils",
       "../modules/audio_processing",

pc/rtp_sender.cc

@@ -297,6 +297,9 @@ void RtpSenderBase::SetSsrc(uint32_t ssrc) {
   if (frame_encryptor_) {
     SetFrameEncryptor(frame_encryptor_);
   }
+  if (frame_transformer_) {
+    SetEncoderToPacketizerFrameTransformer(frame_transformer_);
+  }
 }
 
 void RtpSenderBase::Stop() {
@@ -364,6 +367,17 @@ RTCError RtpSenderBase::DisableEncodingLayers(
   return result;
 }
 
+void RtpSenderBase::SetEncoderToPacketizerFrameTransformer(
+    rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) {
+  frame_transformer_ = std::move(frame_transformer);
+  if (media_channel_ && ssrc_ && !stopped_) {
+    worker_thread_->Invoke<void>(RTC_FROM_HERE, [&] {
+      media_channel_->SetEncoderToPacketizerFrameTransformer(
+          ssrc_, frame_transformer_);
+    });
+  }
+}
+
 LocalAudioSinkAdapter::LocalAudioSinkAdapter() : sink_(nullptr) {}
 
 LocalAudioSinkAdapter::~LocalAudioSinkAdapter() {

pc/rtp_sender.h

@@ -149,6 +149,9 @@ class RtpSenderBase : public RtpSenderInternal, public ObserverInterface {
   // If the specified list is empty, this is a no-op.
   RTCError DisableEncodingLayers(const std::vector<std::string>& rid) override;
 
+  void SetEncoderToPacketizerFrameTransformer(
+      rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) override;
+
  protected:
   // If |set_streams_observer| is not null, it is invoked when SetStreams()
   // is called. |set_streams_observer| is not owned by this object. If not
@@ -197,6 +200,8 @@ class RtpSenderBase : public RtpSenderInternal, public ObserverInterface {
   std::vector<std::string> disabled_rids_;
 
   SetStreamsObserver* set_streams_observer_ = nullptr;
+
+  rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_;
 };
 
 // LocalAudioSinkAdapter receives data callback as a sink to the local

video/video_send_stream_impl.cc

@@ -202,19 +202,20 @@ VideoSendStreamImpl::VideoSendStreamImpl(
       video_stream_encoder_(video_stream_encoder),
       encoder_feedback_(clock, config_->rtp.ssrcs, video_stream_encoder),
       bandwidth_observer_(transport->GetBandwidthObserver()),
-      rtp_video_sender_(transport_->CreateRtpVideoSender(
+      rtp_video_sender_(
-          suspended_ssrcs,
+          transport_->CreateRtpVideoSender(suspended_ssrcs,
-          suspended_payload_states,
+                                           suspended_payload_states,
-          config_->rtp,
+                                           config_->rtp,
-          config_->rtcp_report_interval_ms,
+                                           config_->rtcp_report_interval_ms,
-          config_->send_transport,
+                                           config_->send_transport,
-          CreateObservers(call_stats,
+                                           CreateObservers(call_stats,
-                          &encoder_feedback_,
+                                                           &encoder_feedback_,
-                          stats_proxy_,
+                                                           stats_proxy_,
-                          send_delay_stats),
+                                                           send_delay_stats),
-          event_log,
+                                           event_log,
-          std::move(fec_controller),
+                                           std::move(fec_controller),
-          CreateFrameEncryptionConfig(config_))),
+                                           CreateFrameEncryptionConfig(config_),
+                                           config->frame_transformer)),
       weak_ptr_factory_(this) {
   video_stream_encoder->SetFecControllerOverride(rtp_video_sender_);
   RTC_DCHECK_RUN_ON(worker_queue_);