From 097a4decc2ad2838378827b6a64b7715fda5e955 Mon Sep 17 00:00:00 2001 From: Tony Herre Date: Mon, 19 Jun 2023 15:13:08 +0000 Subject: [PATCH] Make all encodedaudioframes inherit from TransformableAudioFrameI'face Make outgoing encoded audio frames inherit from the same Audio interface that incoming frames inherit from, to align them and make it possible to eg clone frames regardless of their direction. Also begin removing GetHeader() from the Audio interface, replacing it with getters for the specific values we actually need to propagate in the API: sequence number and CSRCs. This makes it much easier to treat incoming and outgoing frames the same, even if they don't have full RtpHeaders prepared at the point of the transform. Bug: chromium:1453226 Change-Id: Ib5b39b30dea8a378b3b26efb1589dfd64741d201 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/308141 Commit-Queue: Tony Herre Reviewed-by: Stefan Holmer Reviewed-by: Palak Agarwal Cr-Commit-Position: refs/heads/main@{#40309} --- api/frame_transformer_factory.cc | 2 +- api/frame_transformer_factory.h | 2 +- api/frame_transformer_interface.h | 8 ++++++ api/test/mock_transformable_audio_frame.h | 8 ++++++ ...nnel_receive_frame_transformer_delegate.cc | 4 +++ ...channel_send_frame_transformer_delegate.cc | 25 +++++++++++++++++-- .../channel_send_frame_transformer_delegate.h | 2 +- 7 files changed, 46 insertions(+), 5 deletions(-) diff --git a/api/frame_transformer_factory.cc b/api/frame_transformer_factory.cc index 262ac0795c..841ab0f941 100644 --- a/api/frame_transformer_factory.cc +++ b/api/frame_transformer_factory.cc @@ -25,7 +25,7 @@ std::unique_ptr CreateVideoReceiverFrame() { return nullptr; } -std::unique_ptr CloneAudioFrame( +std::unique_ptr CloneAudioFrame( TransformableAudioFrameInterface* original) { // At the moment, only making sender frames is supported. return CloneSenderAudioFrame(original); diff --git a/api/frame_transformer_factory.h b/api/frame_transformer_factory.h index e0a7bdf4af..a73ff62956 100644 --- a/api/frame_transformer_factory.h +++ b/api/frame_transformer_factory.h @@ -32,7 +32,7 @@ std::unique_ptr CreateVideoSenderFrame(); std::unique_ptr CreateVideoReceiverFrame(); // Creates a new frame with the same metadata as the original. // The original can be a sender or receiver frame. -RTC_EXPORT std::unique_ptr CloneAudioFrame( +RTC_EXPORT std::unique_ptr CloneAudioFrame( TransformableAudioFrameInterface* original); RTC_EXPORT std::unique_ptr CloneVideoFrame( TransformableVideoFrameInterface* original); diff --git a/api/frame_transformer_interface.h b/api/frame_transformer_interface.h index d4b09a46e2..c47bad4243 100644 --- a/api/frame_transformer_interface.h +++ b/api/frame_transformer_interface.h @@ -72,9 +72,17 @@ class TransformableAudioFrameInterface : public TransformableFrameInterface { // Exposes the frame header, enabling the interface clients to use the // information in the header as needed, for example to compile the list of // csrcs. + // TODO(crbug.com/1453226): Deprecate and remove once callers have migrated to + // the getters for specific fields. virtual const RTPHeader& GetHeader() const = 0; virtual rtc::ArrayView GetContributingSources() const = 0; + + // TODO(crbug.com/1453226): Change this to pure virtual after it + // is implemented everywhere. + virtual const absl::optional SequenceNumber() const { + return absl::nullopt; + } }; // Objects implement this interface to be notified with the transformed frame. diff --git a/api/test/mock_transformable_audio_frame.h b/api/test/mock_transformable_audio_frame.h index 912488adbc..9b52e5c579 100644 --- a/api/test/mock_transformable_audio_frame.h +++ b/api/test/mock_transformable_audio_frame.h @@ -29,6 +29,14 @@ class MockTransformableAudioFrame : public TransformableAudioFrameInterface { GetContributingSources, (), (const override)); + MOCK_METHOD(const absl::optional, + SequenceNumber, + (), + (const, override)); + MOCK_METHOD(TransformableFrameInterface::Direction, + GetDirection, + (), + (const, override)); }; } // namespace webrtc diff --git a/audio/channel_receive_frame_transformer_delegate.cc b/audio/channel_receive_frame_transformer_delegate.cc index 1cb08f0680..8f817c6fe7 100644 --- a/audio/channel_receive_frame_transformer_delegate.cc +++ b/audio/channel_receive_frame_transformer_delegate.cc @@ -46,6 +46,10 @@ class TransformableIncomingAudioFrame } Direction GetDirection() const override { return Direction::kReceiver; } + const absl::optional SequenceNumber() const override { + return header_.sequenceNumber; + } + private: rtc::Buffer payload_; RTPHeader header_; diff --git a/audio/channel_send_frame_transformer_delegate.cc b/audio/channel_send_frame_transformer_delegate.cc index bf7b49527c..93aaa4324d 100644 --- a/audio/channel_send_frame_transformer_delegate.cc +++ b/audio/channel_send_frame_transformer_delegate.cc @@ -15,7 +15,8 @@ namespace webrtc { namespace { -class TransformableOutgoingAudioFrame : public TransformableFrameInterface { +class TransformableOutgoingAudioFrame + : public TransformableAudioFrameInterface { public: TransformableOutgoingAudioFrame(AudioFrameType frame_type, uint8_t payload_type, @@ -50,6 +51,22 @@ class TransformableOutgoingAudioFrame : public TransformableFrameInterface { } Direction GetDirection() const override { return Direction::kSender; } + // TODO(crbug.com/1453226): Remove once GetHeader() is removed from + // TransformableAudioFrameInterface. + const RTPHeader& GetHeader() const override { return empty_header_; } + + rtc::ArrayView GetContributingSources() const override { + return {}; + } + + const absl::optional SequenceNumber() const override { + return absl::nullopt; + } + + void SetRTPTimestamp(uint32_t timestamp) override { + rtp_timestamp_ = timestamp - rtp_start_timestamp_; + } + private: AudioFrameType frame_type_; uint8_t payload_type_; @@ -58,6 +75,10 @@ class TransformableOutgoingAudioFrame : public TransformableFrameInterface { rtc::Buffer payload_; int64_t absolute_capture_timestamp_ms_; uint32_t ssrc_; + + // TODO(crbug.com/1453226): Remove once GetHeader() is removed from + // TransformableAudioFrameInterface. + RTPHeader empty_header_; }; } // namespace @@ -127,7 +148,7 @@ void ChannelSendFrameTransformerDelegate::SendFrame( transformed_frame->GetAbsoluteCaptureTimestampMs()); } -std::unique_ptr CloneSenderAudioFrame( +std::unique_ptr CloneSenderAudioFrame( TransformableAudioFrameInterface* original) { AudioFrameType audio_frame_type = original->GetHeader().extension.voiceActivity diff --git a/audio/channel_send_frame_transformer_delegate.h b/audio/channel_send_frame_transformer_delegate.h index 6bd0af70d7..c866638547 100644 --- a/audio/channel_send_frame_transformer_delegate.h +++ b/audio/channel_send_frame_transformer_delegate.h @@ -78,7 +78,7 @@ class ChannelSendFrameTransformerDelegate : public TransformedFrameCallback { rtc::TaskQueue* encoder_queue_ RTC_GUARDED_BY(send_lock_); }; -std::unique_ptr CloneSenderAudioFrame( +std::unique_ptr CloneSenderAudioFrame( TransformableAudioFrameInterface* original); } // namespace webrtc