Insert audio frame transformer between depacketizer and decoder.

The frame transformer is passed from RTPReceiverInterface through the
library to be eventually set in ChannelReceive, where the frame
transformation will occur in the follow-up CL.

Insertable Streams Web API explainer:
https://github.com/alvestrand/webrtc-media-streams/blob/master/explainer.md

Design doc for WebRTC library changes:
http://doc/1eiLkjNUkRy2FssCPLUp6eH08BZuXXoHfbbBP1ZN7EVk

Bug: webrtc:11380
Change-Id: I5af06d1431047ef50d00e304cf95e92a832b4220
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/171872
Reviewed-by: Magnus Flodman <mflodman@webrtc.org>
Reviewed-by: Tommi <tommi@webrtc.org>
Reviewed-by: Per Åhgren <peah@webrtc.org>
Commit-Queue: Marina Ciocea <marinaciocea@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#30956}

audio/audio_receive_stream.cc

@@ -82,7 +82,8 @@ std::unique_ptr<voe::ChannelReceiveInterface> CreateChannelReceive(
       config.jitter_buffer_max_packets, config.jitter_buffer_fast_accelerate,
       config.jitter_buffer_min_delay_ms,
       config.jitter_buffer_enable_rtx_handling, config.decoder_factory,
-      config.codec_pair_id, config.frame_decryptor, config.crypto_options);
+      config.codec_pair_id, config.frame_decryptor, config.crypto_options,
+      std::move(config.frame_transformer));
 }
 }  // namespace
 
@@ -409,6 +410,12 @@ void AudioReceiveStream::ConfigureStream(AudioReceiveStream* stream,
     channel_receive->SetReceiveCodecs(new_config.decoder_map);
   }
 
+  if (first_time ||
+      old_config.frame_transformer != new_config.frame_transformer) {
+    channel_receive->SetDepacketizerToDecoderFrameTransformer(
+        new_config.frame_transformer);
+  }
+
   stream->config_ = new_config;
 }
 }  // namespace internal

audio/audio_receive_stream_unittest.cc

@@ -100,6 +100,8 @@ struct ConfigHelper {
         .WillRepeatedly(Invoke([](const std::map<int, SdpAudioFormat>& codecs) {
           EXPECT_THAT(codecs, ::testing::IsEmpty());
         }));
+    EXPECT_CALL(*channel_receive_, SetDepacketizerToDecoderFrameTransformer(_))
+        .Times(1);
 
     stream_config_.rtp.local_ssrc = kLocalSsrc;
     stream_config_.rtp.remote_ssrc = kRemoteSsrc;

audio/channel_receive.cc

@@ -20,6 +20,7 @@
 #include <vector>
 
 #include "api/crypto/frame_decryptor_interface.h"
+#include "api/frame_transformer_interface.h"
 #include "api/rtc_event_log/rtc_event_log.h"
 #include "audio/audio_level.h"
 #include "audio/channel_send.h"
@@ -79,7 +80,8 @@ AudioCodingModule::Config AcmConfig(
 class ChannelReceive : public ChannelReceiveInterface {
  public:
   // Used for receive streams.
-  ChannelReceive(Clock* clock,
+  ChannelReceive(
+      Clock* clock,
       ProcessThread* module_process_thread,
       NetEqFactory* neteq_factory,
       AudioDeviceModule* audio_device_module,
@@ -94,7 +96,8 @@ class ChannelReceive : public ChannelReceiveInterface {
       rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
       absl::optional<AudioCodecPairId> codec_pair_id,
       rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
-                 const webrtc::CryptoOptions& crypto_options);
+      const webrtc::CryptoOptions& crypto_options,
+      rtc::scoped_refptr<FrameTransformerInterface> frame_transformer);
   ~ChannelReceive() override;
 
   void SetSink(AudioSinkInterface* sink) override;
@@ -161,6 +164,12 @@ class ChannelReceive : public ChannelReceiveInterface {
   // Used for obtaining RTT for a receive-only channel.
   void SetAssociatedSendChannel(const ChannelSendInterface* channel) override;
 
+  // Sets a frame transformer between the depacketizer and the decoder, to
+  // transform the received frames before decoding them.
+  void SetDepacketizerToDecoderFrameTransformer(
+      rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer)
+      override;
+
  private:
   void ReceivePacket(const uint8_t* packet,
                      size_t packet_length,
@@ -262,6 +271,8 @@ class ChannelReceive : public ChannelReceiveInterface {
   webrtc::CryptoOptions crypto_options_;
 
   webrtc::AbsoluteCaptureTimeReceiver absolute_capture_time_receiver_;
+
+  rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_;
 };
 
 void ChannelReceive::OnReceivedPayloadData(
@@ -422,7 +433,8 @@ ChannelReceive::ChannelReceive(
     rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
     absl::optional<AudioCodecPairId> codec_pair_id,
     rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
-    const webrtc::CryptoOptions& crypto_options)
+    const webrtc::CryptoOptions& crypto_options,
+    rtc::scoped_refptr<FrameTransformerInterface> frame_transformer)
     : event_log_(rtc_event_log),
       rtp_receive_statistics_(ReceiveStatistics::Create(clock)),
       remote_ssrc_(remote_ssrc),
@@ -444,7 +456,8 @@ ChannelReceive::ChannelReceive(
       associated_send_channel_(nullptr),
       frame_decryptor_(frame_decryptor),
       crypto_options_(crypto_options),
-      absolute_capture_time_receiver_(clock) {
+      absolute_capture_time_receiver_(clock),
+      frame_transformer_(std::move(frame_transformer)) {
   // TODO(nisse): Use _moduleProcessThreadPtr instead?
   module_process_thread_checker_.Detach();
 
@@ -742,6 +755,12 @@ void ChannelReceive::SetAssociatedSendChannel(
   associated_send_channel_ = channel;
 }
 
+void ChannelReceive::SetDepacketizerToDecoderFrameTransformer(
+    rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
+  RTC_DCHECK(worker_thread_checker_.IsCurrent());
+  frame_transformer_ = std::move(frame_transformer);
+}
+
 NetworkStatistics ChannelReceive::GetNetworkStatistics() const {
   RTC_DCHECK(worker_thread_checker_.IsCurrent());
   NetworkStatistics stats;
@@ -927,13 +946,15 @@ std::unique_ptr<ChannelReceiveInterface> CreateChannelReceive(
     rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
     absl::optional<AudioCodecPairId> codec_pair_id,
     rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
-    const webrtc::CryptoOptions& crypto_options) {
+    const webrtc::CryptoOptions& crypto_options,
+    rtc::scoped_refptr<FrameTransformerInterface> frame_transformer) {
   return std::make_unique<ChannelReceive>(
       clock, module_process_thread, neteq_factory, audio_device_module,
       rtcp_send_transport, rtc_event_log, local_ssrc, remote_ssrc,
       jitter_buffer_max_packets, jitter_buffer_fast_playout,
       jitter_buffer_min_delay_ms, jitter_buffer_enable_rtx_handling,
-      decoder_factory, codec_pair_id, frame_decryptor, crypto_options);
+      decoder_factory, codec_pair_id, frame_decryptor, crypto_options,
+      std::move(frame_transformer));
 }
 
 }  // namespace voe

audio/channel_receive.h

@@ -22,6 +22,7 @@
 #include "api/call/audio_sink.h"
 #include "api/call/transport.h"
 #include "api/crypto/crypto_options.h"
+#include "api/frame_transformer_interface.h"
 #include "api/neteq/neteq_factory.h"
 #include "api/transport/rtp/rtp_source.h"
 #include "call/rtp_packet_sink_interface.h"
@@ -137,6 +138,12 @@ class ChannelReceiveInterface : public RtpPacketSinkInterface {
   // Used for obtaining RTT for a receive-only channel.
   virtual void SetAssociatedSendChannel(
       const ChannelSendInterface* channel) = 0;
+
+  // Sets a frame transformer between the depacketizer and the decoder, to
+  // transform the received frames before decoding them.
+  virtual void SetDepacketizerToDecoderFrameTransformer(
+      rtc::scoped_refptr<webrtc::FrameTransformerInterface>
+          frame_transformer) = 0;
 };
 
 std::unique_ptr<ChannelReceiveInterface> CreateChannelReceive(
@@ -155,7 +162,8 @@ std::unique_ptr<ChannelReceiveInterface> CreateChannelReceive(
     rtc::scoped_refptr<AudioDecoderFactory> decoder_factory,
     absl::optional<AudioCodecPairId> codec_pair_id,
     rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor,
-    const webrtc::CryptoOptions& crypto_options);
+    const webrtc::CryptoOptions& crypto_options,
+    rtc::scoped_refptr<FrameTransformerInterface> frame_transformer);
 
 }  // namespace voe
 }  // namespace webrtc

audio/mock_voe_channel_proxy.h

@@ -66,6 +66,9 @@ class MockChannelReceive : public voe::ChannelReceiveInterface {
   MOCK_CONST_METHOD0(GetSources, std::vector<RtpSource>());
   MOCK_METHOD0(StartPlayout, void());
   MOCK_METHOD0(StopPlayout, void());
+  MOCK_METHOD1(SetDepacketizerToDecoderFrameTransformer,
+               void(rtc::scoped_refptr<webrtc::FrameTransformerInterface>
+                        frame_transformer));
 };
 
 class MockChannelSend : public voe::ChannelSendInterface {

call/audio_receive_stream.h

@@ -21,6 +21,7 @@
 #include "api/call/transport.h"
 #include "api/crypto/crypto_options.h"
 #include "api/crypto/frame_decryptor_interface.h"
+#include "api/frame_transformer_interface.h"
 #include "api/rtp_parameters.h"
 #include "api/scoped_refptr.h"
 #include "api/transport/rtp/rtp_source.h"
@@ -150,6 +151,10 @@ class AudioReceiveStream {
     // decrypted in whatever way the caller choses. This is not required by
     // default.
     rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor;
+
+    // An optional frame transformer used by insertable streams to transform
+    // encoded frames.
+    rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer;
   };
 
   // Reconfigure the stream according to the Configuration.

media/engine/webrtc_voice_engine.cc

@@ -1063,7 +1063,8 @@ class WebRtcVoiceMediaChannel::WebRtcAudioReceiveStream {
       int jitter_buffer_min_delay_ms,
       bool jitter_buffer_enable_rtx_handling,
       rtc::scoped_refptr<webrtc::FrameDecryptorInterface> frame_decryptor,
-      const webrtc::CryptoOptions& crypto_options)
+      const webrtc::CryptoOptions& crypto_options,
+      rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer)
       : call_(call), config_() {
     RTC_DCHECK(call);
     config_.rtp.remote_ssrc = remote_ssrc;
@@ -1085,6 +1086,7 @@ class WebRtcVoiceMediaChannel::WebRtcAudioReceiveStream {
     config_.codec_pair_id = codec_pair_id;
     config_.frame_decryptor = frame_decryptor;
     config_.crypto_options = crypto_options;
+    config_.frame_transformer = std::move(frame_transformer);
     RecreateAudioReceiveStream();
   }
 
@@ -1215,6 +1217,13 @@ class WebRtcVoiceMediaChannel::WebRtcAudioReceiveStream {
     return rtp_parameters;
   }
 
+  void SetDepacketizerToDecoderFrameTransformer(
+      rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
+    RTC_DCHECK(worker_thread_checker_.IsCurrent());
+    config_.frame_transformer = std::move(frame_transformer);
+    ReconfigureAudioReceiveStream();
+  }
+
  private:
   void RecreateAudioReceiveStream() {
     RTC_DCHECK(worker_thread_checker_.IsCurrent());
@@ -1868,7 +1877,7 @@ bool WebRtcVoiceMediaChannel::AddRecvStream(const StreamParams& sp) {
                 engine()->audio_jitter_buffer_fast_accelerate_,
                 engine()->audio_jitter_buffer_min_delay_ms_,
                 engine()->audio_jitter_buffer_enable_rtx_handling_,
-                unsignaled_frame_decryptor_, crypto_options_)));
+                unsignaled_frame_decryptor_, crypto_options_, nullptr)));
   recv_streams_[ssrc]->SetPlayout(playout_);
 
   return true;
@@ -2337,6 +2346,20 @@ void WebRtcVoiceMediaChannel::SetEncoderToPacketizerFrameTransformer(
       std::move(frame_transformer));
 }
 
+void WebRtcVoiceMediaChannel::SetDepacketizerToDecoderFrameTransformer(
+    uint32_t ssrc,
+    rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
+  RTC_DCHECK(worker_thread_checker_.IsCurrent());
+  auto matching_stream = recv_streams_.find(ssrc);
+  if (matching_stream == recv_streams_.end()) {
+    RTC_LOG(LS_INFO) << "Attempting to set frame transformer for SSRC:" << ssrc
+                     << " which doesn't exist.";
+    return;
+  }
+  matching_stream->second->SetDepacketizerToDecoderFrameTransformer(
+      std::move(frame_transformer));
+}
+
 bool WebRtcVoiceMediaChannel::MaybeDeregisterUnsignaledRecvStream(
     uint32_t ssrc) {
   RTC_DCHECK(worker_thread_checker_.IsCurrent());

media/engine/webrtc_voice_engine.h

@@ -215,6 +215,10 @@ class WebRtcVoiceMediaChannel final : public VoiceMediaChannel,
       uint32_t ssrc,
       rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer)
       override;
+  void SetDepacketizerToDecoderFrameTransformer(
+      uint32_t ssrc,
+      rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer)
+      override;
 
   // implements Transport interface
   bool SendRtp(const uint8_t* data,

pc/audio_rtp_receiver.cc

@@ -225,6 +225,20 @@ std::vector<RtpSource> AudioRtpReceiver::GetSources() const {
       RTC_FROM_HERE, [&] { return media_channel_->GetSources(*ssrc_); });
 }
 
+void AudioRtpReceiver::SetDepacketizerToDecoderFrameTransformer(
+    rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer) {
+  if (media_channel_ && ssrc_.has_value() && !stopped_) {
+    worker_thread_->Invoke<void>(
+        RTC_FROM_HERE,
+        [this, frame_transformer = std::move(frame_transformer)] {
+          RTC_DCHECK_RUN_ON(worker_thread_);
+          frame_transformer_ = frame_transformer;
+          media_channel_->SetDepacketizerToDecoderFrameTransformer(
+              *ssrc_, frame_transformer);
+        });
+  }
+}
+
 void AudioRtpReceiver::Reconfigure() {
   if (!media_channel_ || stopped_) {
     RTC_LOG(LS_ERROR)
@@ -237,6 +251,16 @@ void AudioRtpReceiver::Reconfigure() {
   // Reattach the frame decryptor if we were reconfigured.
   MaybeAttachFrameDecryptorToMediaChannel(
       ssrc_, worker_thread_, frame_decryptor_, media_channel_, stopped_);
+
+  if (media_channel_ && ssrc_.has_value() && !stopped_) {
+    worker_thread_->Invoke<void>(RTC_FROM_HERE, [this] {
+      RTC_DCHECK_RUN_ON(worker_thread_);
+      if (!frame_transformer_)
+        return;
+      media_channel_->SetDepacketizerToDecoderFrameTransformer(
+          *ssrc_, frame_transformer_);
+    });
+  }
 }
 
 void AudioRtpReceiver::SetObserver(RtpReceiverObserverInterface* observer) {

pc/audio_rtp_receiver.h

@@ -104,6 +104,9 @@ class AudioRtpReceiver : public ObserverInterface,
 
   std::vector<RtpSource> GetSources() const override;
   int AttachmentId() const override { return attachment_id_; }
+  void SetDepacketizerToDecoderFrameTransformer(
+      rtc::scoped_refptr<webrtc::FrameTransformerInterface> frame_transformer)
+      override;
 
  private:
   void RestartMediaChannel(absl::optional<uint32_t> ssrc);
@@ -128,6 +131,8 @@ class AudioRtpReceiver : public ObserverInterface,
   // Allows to thread safely change playout delay. Handles caching cases if
   // |SetJitterBufferMinimumDelay| is called before start.
   rtc::scoped_refptr<JitterBufferDelayInterface> delay_;
+  rtc::scoped_refptr<FrameTransformerInterface> frame_transformer_
+      RTC_GUARDED_BY(worker_thread_);
 };
 
 }  // namespace webrtc