diff --git a/audio/BUILD.gn b/audio/BUILD.gn index d4ccd42fb5..7da3c79677 100644 --- a/audio/BUILD.gn +++ b/audio/BUILD.gn @@ -38,6 +38,7 @@ rtc_library("audio") { deps = [ "../api:array_view", "../api:call_api", + "../api:frame_transformer_interface", "../api:function_view", "../api:rtp_headers", "../api:rtp_parameters", @@ -84,6 +85,7 @@ rtc_library("audio") { "../rtc_base:rtc_task_queue", "../rtc_base:safe_minmax", "../rtc_base/experiments:field_trial_parser", + "../rtc_base/synchronization:sequence_checker", "../system_wrappers", "../system_wrappers:field_trial", "../system_wrappers:metrics", diff --git a/audio/audio_send_stream.cc b/audio/audio_send_stream.cc index 2dcb37adf1..acdc73b9a9 100644 --- a/audio/audio_send_stream.cc +++ b/audio/audio_send_stream.cc @@ -127,7 +127,8 @@ AudioSendStream::AudioSendStream( config.crypto_options, config.rtp.extmap_allow_mixed, config.rtcp_report_interval_ms, - config.rtp.ssrc)) {} + config.rtp.ssrc, + config.frame_transformer)) {} AudioSendStream::AudioSendStream( Clock* clock, @@ -249,6 +250,12 @@ void AudioSendStream::ConfigureStream( channel_send_->SetFrameEncryptor(new_config.frame_encryptor); } + if (first_time || + new_config.frame_transformer != old_config.frame_transformer) { + channel_send_->SetEncoderToPacketizerFrameTransformer( + new_config.frame_transformer); + } + if (first_time || new_config.rtp.extmap_allow_mixed != old_config.rtp.extmap_allow_mixed) { rtp_rtcp_module_->SetExtmapAllowMixed(new_config.rtp.extmap_allow_mixed); diff --git a/audio/audio_send_stream_unittest.cc b/audio/audio_send_stream_unittest.cc index de152cdbcd..de1f2fe007 100644 --- a/audio/audio_send_stream_unittest.cc +++ b/audio/audio_send_stream_unittest.cc @@ -215,6 +215,8 @@ struct ConfigHelper { EXPECT_CALL(rtp_rtcp_, SSRC).WillRepeatedly(Return(kSsrc)); EXPECT_CALL(*channel_send_, SetRTCP_CNAME(StrEq(kCName))).Times(1); EXPECT_CALL(*channel_send_, SetFrameEncryptor(_)).Times(1); + EXPECT_CALL(*channel_send_, SetEncoderToPacketizerFrameTransformer(_)) + .Times(1); EXPECT_CALL(rtp_rtcp_, SetExtmapAllowMixed(false)).Times(1); EXPECT_CALL(*channel_send_, SetSendAudioLevelIndicationStatus(true, kAudioLevelId)) diff --git a/audio/channel_send.cc b/audio/channel_send.cc index dd866f3f7b..d8ac39c8e2 100644 --- a/audio/channel_send.cc +++ b/audio/channel_send.cc @@ -76,7 +76,8 @@ class ChannelSend : public ChannelSendInterface, const webrtc::CryptoOptions& crypto_options, bool extmap_allow_mixed, int rtcp_report_interval_ms, - uint32_t ssrc); + uint32_t ssrc, + rtc::scoped_refptr frame_transformer); ~ChannelSend() override; @@ -142,6 +143,12 @@ class ChannelSend : public ChannelSendInterface, void SetFrameEncryptor( rtc::scoped_refptr frame_encryptor) override; + // Sets a frame transformer between encoder and packetizer, to transform + // encoded frames before sending them out the network. + void SetEncoderToPacketizerFrameTransformer( + rtc::scoped_refptr frame_transformer) + override; + private: // From AudioPacketizationCallback in the ACM int32_t SendData(AudioFrameType frameType, @@ -217,6 +224,10 @@ class ChannelSend : public ChannelSendInterface, // E2EE Frame Encryption Options const webrtc::CryptoOptions crypto_options_; + // Frame transformer used by insertable streams to transform encoded frames. + rtc::scoped_refptr frame_transformer_ + RTC_GUARDED_BY(encoder_queue_); + rtc::CriticalSection bitrate_crit_section_; int configured_bitrate_bps_ RTC_GUARDED_BY(bitrate_crit_section_) = 0; @@ -452,18 +463,20 @@ int32_t ChannelSend::SendRtpAudio(AudioFrameType frameType, return 0; } -ChannelSend::ChannelSend(Clock* clock, - TaskQueueFactory* task_queue_factory, - ProcessThread* module_process_thread, - OverheadObserver* overhead_observer, - Transport* rtp_transport, - RtcpRttStats* rtcp_rtt_stats, - RtcEventLog* rtc_event_log, - FrameEncryptorInterface* frame_encryptor, - const webrtc::CryptoOptions& crypto_options, - bool extmap_allow_mixed, - int rtcp_report_interval_ms, - uint32_t ssrc) +ChannelSend::ChannelSend( + Clock* clock, + TaskQueueFactory* task_queue_factory, + ProcessThread* module_process_thread, + OverheadObserver* overhead_observer, + Transport* rtp_transport, + RtcpRttStats* rtcp_rtt_stats, + RtcEventLog* rtc_event_log, + FrameEncryptorInterface* frame_encryptor, + const webrtc::CryptoOptions& crypto_options, + bool extmap_allow_mixed, + int rtcp_report_interval_ms, + uint32_t ssrc, + rtc::scoped_refptr frame_transformer) : event_log_(rtc_event_log), _timeStamp(0), // This is just an offset, RTP module will add it's own // random offset @@ -478,6 +491,7 @@ ChannelSend::ChannelSend(Clock* clock, new RateLimiter(clock, kMaxRetransmissionWindowMs)), frame_encryptor_(frame_encryptor), crypto_options_(crypto_options), + frame_transformer_(std::move(frame_transformer)), encoder_queue_(task_queue_factory->CreateTaskQueue( "AudioEncoder", TaskQueueFactory::Priority::NORMAL)) { @@ -898,6 +912,16 @@ void ChannelSend::SetFrameEncryptor( }); } +void ChannelSend::SetEncoderToPacketizerFrameTransformer( + rtc::scoped_refptr frame_transformer) { + RTC_DCHECK_RUN_ON(&worker_thread_checker_); + encoder_queue_.PostTask( + [this, frame_transformer = std::move(frame_transformer)]() mutable { + RTC_DCHECK_RUN_ON(&encoder_queue_); + frame_transformer_ = std::move(frame_transformer); + }); +} + void ChannelSend::OnReceivedRtt(int64_t rtt_ms) { // Invoke audio encoders OnReceivedRtt(). CallEncoder( @@ -918,11 +942,13 @@ std::unique_ptr CreateChannelSend( const webrtc::CryptoOptions& crypto_options, bool extmap_allow_mixed, int rtcp_report_interval_ms, - uint32_t ssrc) { + uint32_t ssrc, + rtc::scoped_refptr frame_transformer) { return std::make_unique( clock, task_queue_factory, module_process_thread, overhead_observer, rtp_transport, rtcp_rtt_stats, rtc_event_log, frame_encryptor, - crypto_options, extmap_allow_mixed, rtcp_report_interval_ms, ssrc); + crypto_options, extmap_allow_mixed, rtcp_report_interval_ms, ssrc, + std::move(frame_transformer)); } } // namespace voe diff --git a/audio/channel_send.h b/audio/channel_send.h index 0fe434b684..94c554015e 100644 --- a/audio/channel_send.h +++ b/audio/channel_send.h @@ -18,6 +18,7 @@ #include "api/audio/audio_frame.h" #include "api/audio_codecs/audio_encoder.h" #include "api/crypto/crypto_options.h" +#include "api/frame_transformer_interface.h" #include "api/function_view.h" #include "api/task_queue/task_queue_factory.h" #include "modules/rtp_rtcp/include/report_block_data.h" @@ -115,6 +116,12 @@ class ChannelSendInterface { // E2EE Custom Audio Frame Encryption (Optional) virtual void SetFrameEncryptor( rtc::scoped_refptr frame_encryptor) = 0; + + // Sets a frame transformer between encoder and packetizer, to transform + // encoded frames before sending them out the network. + virtual void SetEncoderToPacketizerFrameTransformer( + rtc::scoped_refptr + frame_transformer) = 0; }; std::unique_ptr CreateChannelSend( @@ -129,7 +136,8 @@ std::unique_ptr CreateChannelSend( const webrtc::CryptoOptions& crypto_options, bool extmap_allow_mixed, int rtcp_report_interval_ms, - uint32_t ssrc); + uint32_t ssrc, + rtc::scoped_refptr frame_transformer); } // namespace voe } // namespace webrtc diff --git a/audio/mock_voe_channel_proxy.h b/audio/mock_voe_channel_proxy.h index e4c60a1a4b..9a013ff03f 100644 --- a/audio/mock_voe_channel_proxy.h +++ b/audio/mock_voe_channel_proxy.h @@ -118,6 +118,9 @@ class MockChannelSend : public voe::ChannelSendInterface { MOCK_METHOD1( SetFrameEncryptor, void(rtc::scoped_refptr frame_encryptor)); + MOCK_METHOD1(SetEncoderToPacketizerFrameTransformer, + void(rtc::scoped_refptr + frame_transformer)); }; } // namespace test } // namespace webrtc diff --git a/call/BUILD.gn b/call/BUILD.gn index 3ef7bfa2b6..808aa73afc 100644 --- a/call/BUILD.gn +++ b/call/BUILD.gn @@ -32,6 +32,7 @@ rtc_library("call_interfaces") { ":rtp_interfaces", ":video_stream_api", "../api:fec_controller_api", + "../api:frame_transformer_interface", "../api:network_state_predictor_api", "../api:rtc_error", "../api:rtp_headers", diff --git a/call/audio_send_stream.h b/call/audio_send_stream.h index e60bfcdc12..86cea38938 100644 --- a/call/audio_send_stream.h +++ b/call/audio_send_stream.h @@ -23,6 +23,7 @@ #include "api/call/transport.h" #include "api/crypto/crypto_options.h" #include "api/crypto/frame_encryptor_interface.h" +#include "api/frame_transformer_interface.h" #include "api/rtp_parameters.h" #include "api/scoped_refptr.h" #include "call/audio_sender.h" @@ -157,6 +158,10 @@ class AudioSendStream : public AudioSender { // encryptor in whatever way the caller choses. This is not required by // default. rtc::scoped_refptr frame_encryptor; + + // An optional frame transformer used by insertable streams to transform + // encoded frames. + rtc::scoped_refptr frame_transformer; }; virtual ~AudioSendStream() = default; diff --git a/media/engine/webrtc_voice_engine.cc b/media/engine/webrtc_voice_engine.cc index 80055e718e..972a7ac837 100644 --- a/media/engine/webrtc_voice_engine.cc +++ b/media/engine/webrtc_voice_engine.cc @@ -949,6 +949,13 @@ class WebRtcVoiceMediaChannel::WebRtcAudioSendStream return webrtc::RTCError::OK(); } + void SetEncoderToPacketizerFrameTransformer( + rtc::scoped_refptr frame_transformer) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + config_.frame_transformer = std::move(frame_transformer); + ReconfigureAudioSendStream(); + } + private: void UpdateSendState() { RTC_DCHECK(worker_thread_checker_.IsCurrent()); @@ -2316,6 +2323,20 @@ std::vector WebRtcVoiceMediaChannel::GetSources( return it->second->GetSources(); } +void WebRtcVoiceMediaChannel::SetEncoderToPacketizerFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr frame_transformer) { + RTC_DCHECK(worker_thread_checker_.IsCurrent()); + auto matching_stream = send_streams_.find(ssrc); + if (matching_stream == send_streams_.end()) { + RTC_LOG(LS_INFO) << "Attempting to set frame transformer for SSRC:" << ssrc + << " which doesn't exist."; + return; + } + matching_stream->second->SetEncoderToPacketizerFrameTransformer( + std::move(frame_transformer)); +} + bool WebRtcVoiceMediaChannel::MaybeDeregisterUnsignaledRecvStream( uint32_t ssrc) { RTC_DCHECK(worker_thread_checker_.IsCurrent()); diff --git a/media/engine/webrtc_voice_engine.h b/media/engine/webrtc_voice_engine.h index 01b7123c7a..a1f8ff5b04 100644 --- a/media/engine/webrtc_voice_engine.h +++ b/media/engine/webrtc_voice_engine.h @@ -209,6 +209,13 @@ class WebRtcVoiceMediaChannel final : public VoiceMediaChannel, std::vector GetSources(uint32_t ssrc) const override; + // Sets a frame transformer between encoder and packetizer, to transform + // encoded frames before sending them out the network. + void SetEncoderToPacketizerFrameTransformer( + uint32_t ssrc, + rtc::scoped_refptr frame_transformer) + override; + // implements Transport interface bool SendRtp(const uint8_t* data, size_t len,