mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-13 05:40:42 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Expose audio mimeType for insertable streams

Browse source 
Split from
  https://webrtc-review.googlesource.com/c/src/+/318283
to reduce CL size. Takes a different and (hopefully) simpler
approach.

BUG=webrtc:15579

Change-Id: I8517ffbeb0f0a76db80e3e367de727fb6976211d
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/325023
Commit-Queue: Philipp Hancke <phancke@microsoft.com>
Reviewed-by: Harald Alvestrand <hta@webrtc.org>
Reviewed-by: Tony Herre <herre@google.com>
Cr-Commit-Position: refs/heads/main@{#41073}
This commit is contained in:
Philipp Hancke 2023-11-01 19:18:32 +01:00 committed by WebRTC LUCI CQ
parent 23cecc1d43
commit d2098933e1
13 changed files with 74 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
audio/audio_send_stream.cc
View file

@ -636,12 +636,14 @@ bool AudioSendStream::SetupSendCodec(const Config& new_config) {
}
// Wrap the encoder in a RED encoder, if RED is enabled.
SdpAudioFormat format = spec.format;
if (spec.red_payload_type) {
AudioEncoderCopyRed::Config red_config;
red_config.payload_type = *spec.red_payload_type;
red_config.speech_encoder = std::move(encoder);
encoder = std::make_unique<AudioEncoderCopyRed>(std::move(red_config),
field_trials_);
format.name = cricket::kRedCodecName;
}
// Set currently known overhead (used in ANA, opus only).
@ -655,7 +657,7 @@ bool AudioSendStream::SetupSendCodec(const Config& new_config) {
}
StoreEncoderProperties(encoder->SampleRateHz(), encoder->NumChannels());
channel_send_->SetEncoder(new_config.send_codec_spec->payload_type,
channel_send_->SetEncoder(new_config.send_codec_spec->payload_type, format,
std::move(encoder));
return true;

11
audio/audio_send_stream_unittest.cc
View file

@ -242,11 +242,11 @@ struct ConfigHelper {
void SetupMockForSetupSendCodec(bool expect_set_encoder_call) {
if (expect_set_encoder_call) {
EXPECT_CALL(*channel_send_, SetEncoder)
.WillOnce(
[this](int payload_type, std::unique_ptr<AudioEncoder> encoder) {
this->audio_encoder_ = std::move(encoder);
return true;
});
.WillOnce([this](int payload_type, const SdpAudioFormat& format,
std::unique_ptr<AudioEncoder> encoder) {
this->audio_encoder_ = std::move(encoder);
return true;
});
}
}
@ -595,6 +595,7 @@ TEST(AudioSendStreamTest, SendCodecCanApplyVad) {
std::unique_ptr<AudioEncoder> stolen_encoder;
EXPECT_CALL(*helper.channel_send(), SetEncoder)
.WillOnce([&stolen_encoder](int payload_type,
const SdpAudioFormat& format,
std::unique_ptr<AudioEncoder> encoder) {
stolen_encoder = std::move(encoder);
return true;

13
audio/channel_receive.cc
View file

@ -47,6 +47,7 @@
#include "rtc_base/numerics/safe_minmax.h"
#include "rtc_base/numerics/sequence_number_unwrapper.h"
#include "rtc_base/race_checker.h"
#include "rtc_base/strings/string_builder.h"
#include "rtc_base/synchronization/mutex.h"
#include "rtc_base/system/no_unique_address.h"
#include "rtc_base/time_utils.h"
@ -312,6 +313,8 @@ class ChannelReceive : public ChannelReceiveInterface,
mutable Mutex rtcp_counter_mutex_;
RtcpPacketTypeCounter rtcp_packet_type_counter_
RTC_GUARDED_BY(rtcp_counter_mutex_);
std::map<int, SdpAudioFormat> payload_type_map_;
};
void ChannelReceive::OnReceivedPayloadData(
@ -636,6 +639,7 @@ void ChannelReceive::SetReceiveCodecs(
RTC_DCHECK_GE(kv.second.clockrate_hz, 1000);
payload_type_frequencies_[kv.first] = kv.second.clockrate_hz;
}
payload_type_map_ = codecs;
acm_receiver_.SetCodecs(codecs);
}
@ -722,7 +726,14 @@ void ChannelReceive::ReceivePacket(const uint8_t* packet,
if (frame_transformer_delegate_) {
// Asynchronously transform the received payload. After the payload is
// transformed, the delegate will call OnReceivedPayloadData to handle it.
frame_transformer_delegate_->Transform(payload_data, header, remote_ssrc_);
char buf[1024];
rtc::SimpleStringBuilder mime_type(buf);
auto it = payload_type_map_.find(header.payloadType);
mime_type << MediaTypeToString(cricket::MEDIA_TYPE_AUDIO) << "/"
<< (it != payload_type_map_.end() ? it->second.name
: "x-unknown");
frame_transformer_delegate_->Transform(payload_data, header, remote_ssrc_,
mime_type.str());
} else {
OnReceivedPayloadData(payload_data, header);
}

15
audio/channel_receive_frame_transformer_delegate.cc
View file

@ -10,6 +10,7 @@
#include "audio/channel_receive_frame_transformer_delegate.h"
#include <string>
#include <utility>
#include "rtc_base/buffer.h"
@ -22,10 +23,12 @@ class TransformableIncomingAudioFrame
public:
TransformableIncomingAudioFrame(rtc::ArrayView<const uint8_t> payload,
const RTPHeader& header,
uint32_t ssrc)
uint32_t ssrc,
const std::string& codec_mime_type)
: payload_(payload.data(), payload.size()),
header_(header),
ssrc_(ssrc) {}
ssrc_(ssrc),
codec_mime_type_(codec_mime_type) {}
~TransformableIncomingAudioFrame() override = default;
rtc::ArrayView<const uint8_t> GetData() const override { return payload_; }
@ -45,6 +48,7 @@ class TransformableIncomingAudioFrame
}
Direction GetDirection() const override { return Direction::kReceiver; }
std::string GetMimeType() const override { return codec_mime_type_; }
const absl::optional<uint16_t> SequenceNumber() const override {
return header_.sequenceNumber;
}
@ -65,6 +69,7 @@ class TransformableIncomingAudioFrame
rtc::Buffer payload_;
RTPHeader header_;
uint32_t ssrc_;
std::string codec_mime_type_;
};
} // namespace
@ -92,10 +97,12 @@ void ChannelReceiveFrameTransformerDelegate::Reset() {
void ChannelReceiveFrameTransformerDelegate::Transform(
rtc::ArrayView<const uint8_t> packet,
const RTPHeader& header,
uint32_t ssrc) {
uint32_t ssrc,
const std::string& codec_mime_type) {
RTC_DCHECK_RUN_ON(&sequence_checker_);
frame_transformer_->Transform(
std::make_unique<TransformableIncomingAudioFrame>(packet, header, ssrc));
std::make_unique<TransformableIncomingAudioFrame>(packet, header, ssrc,
codec_mime_type));
}
void ChannelReceiveFrameTransformerDelegate::OnTransformedFrame(

4
audio/channel_receive_frame_transformer_delegate.h
View file

@ -12,6 +12,7 @@
#define AUDIO_CHANNEL_RECEIVE_FRAME_TRANSFORMER_DELEGATE_H_
#include <memory>
#include <string>
#include "api/frame_transformer_interface.h"
#include "api/sequence_checker.h"
@ -48,7 +49,8 @@ class ChannelReceiveFrameTransformerDelegate : public TransformedFrameCallback {
// the frame asynchronously.
void Transform(rtc::ArrayView<const uint8_t> packet,
const RTPHeader& header,
uint32_t ssrc);
uint32_t ssrc,
const std::string& codec_mime_type);
// Implements TransformedFrameCallback. Can be called on any thread.
void OnTransformedFrame(

4
audio/channel_receive_frame_transformer_delegate_unittest.cc
View file

@ -93,7 +93,7 @@ TEST(ChannelReceiveFrameTransformerDelegateTest,
[&callback](std::unique_ptr<TransformableFrameInterface> frame) {
callback->OnTransformedFrame(std::move(frame));
});
delegate->Transform(packet, header, 1111 /*ssrc*/);
delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus");
rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
}
@ -126,7 +126,7 @@ TEST(ChannelReceiveFrameTransformerDelegateTest,
static_cast<TransformableAudioFrameInterface*>(frame.get());
callback->OnTransformedFrame(CloneSenderAudioFrame(transformed_frame));
});
delegate->Transform(packet, header, 1111 /*ssrc*/);
delegate->Transform(packet, header, /*ssrc=*/1111, /*mimeType=*/"audio/opus");
rtc::ThreadManager::ProcessAllMessageQueuesForTesting();
}

15
audio/channel_send.cc
View file

@ -37,6 +37,7 @@
#include "rtc_base/numerics/safe_conversions.h"
#include "rtc_base/race_checker.h"
#include "rtc_base/rate_limiter.h"
#include "rtc_base/strings/string_builder.h"
#include "rtc_base/synchronization/mutex.h"
#include "rtc_base/task_queue.h"
#include "rtc_base/time_utils.h"
@ -79,6 +80,7 @@ class ChannelSend : public ChannelSendInterface,
// Send using this encoder, with this payload type.
void SetEncoder(int payload_type,
const SdpAudioFormat& encoder_format,
std::unique_ptr<AudioEncoder> encoder) override;
void ModifyEncoder(rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)>
modifier) override;
@ -237,6 +239,8 @@ class ChannelSend : public ChannelSendInterface,
// Defined last to ensure that there are no running tasks when the other
// members are destroyed.
rtc::TaskQueue encoder_queue_;
SdpAudioFormat encoder_format_;
};
const int kTelephoneEventAttenuationdB = 10;
@ -279,10 +283,14 @@ int32_t ChannelSend::SendData(AudioFrameType frameType,
if (frame_transformer_delegate_) {
// Asynchronously transform the payload before sending it. After the payload
// is transformed, the delegate will call SendRtpAudio to send it.
char buf[1024];
rtc::SimpleStringBuilder mime_type(buf);
mime_type << MediaTypeToString(cricket::MEDIA_TYPE_AUDIO) << "/"
<< encoder_format_.name;
frame_transformer_delegate_->Transform(
frameType, payloadType, rtp_timestamp + rtp_rtcp_->StartTimestamp(),
payloadData, payloadSize, absolute_capture_timestamp_ms,
rtp_rtcp_->SSRC());
rtp_rtcp_->SSRC(), mime_type.str());
return 0;
}
return SendRtpAudio(frameType, payloadType, rtp_timestamp, payload,
@ -394,7 +402,8 @@ ChannelSend::ChannelSend(
crypto_options_(crypto_options),
encoder_queue_(task_queue_factory->CreateTaskQueue(
"AudioEncoder",
TaskQueueFactory::Priority::NORMAL)) {
TaskQueueFactory::Priority::NORMAL)),
encoder_format_("x-unknown", 0, 0) {
audio_coding_ = AudioCodingModule::Create();
RtpRtcpInterface::Configuration configuration;
@ -495,6 +504,7 @@ void ChannelSend::StopSend() {
}
void ChannelSend::SetEncoder(int payload_type,
const SdpAudioFormat& encoder_format,
std::unique_ptr<AudioEncoder> encoder) {
RTC_DCHECK_RUN_ON(&worker_thread_checker_);
RTC_DCHECK_GE(payload_type, 0);
@ -508,6 +518,7 @@ void ChannelSend::SetEncoder(int payload_type,
encoder->RtpTimestampRateHz(),
encoder->NumChannels(), 0);
encoder_format_ = encoder_format;
audio_coding_->SetEncoder(std::move(encoder));
}

1
audio/channel_send.h
View file

@ -62,6 +62,7 @@ class ChannelSendInterface {
virtual CallSendStatistics GetRTCPStatistics() const = 0;
virtual void SetEncoder(int payload_type,
const SdpAudioFormat& encoder_format,
std::unique_ptr<AudioEncoder> encoder) = 0;
virtual void ModifyEncoder(
rtc::FunctionView<void(std::unique_ptr<AudioEncoder>*)> modifier) = 0;

16
audio/channel_send_frame_transformer_delegate.cc
View file

@ -55,13 +55,15 @@ class TransformableOutgoingAudioFrame
const uint8_t* payload_data,
size_t payload_size,
absl::optional<uint64_t> absolute_capture_timestamp_ms,
uint32_t ssrc)
uint32_t ssrc,
const std::string& codec_mime_type)
: frame_type_(frame_type),
payload_type_(payload_type),
rtp_timestamp_with_offset_(rtp_timestamp_with_offset),
payload_(payload_data, payload_size),
absolute_capture_timestamp_ms_(absolute_capture_timestamp_ms),
ssrc_(ssrc) {}
ssrc_(ssrc),
codec_mime_type_(codec_mime_type) {}
~TransformableOutgoingAudioFrame() override = default;
rtc::ArrayView<const uint8_t> GetData() const override { return payload_; }
void SetData(rtc::ArrayView<const uint8_t> data) override {
@ -76,6 +78,7 @@ class TransformableOutgoingAudioFrame
uint8_t GetPayloadType() const override { return payload_type_; }
Direction GetDirection() const override { return Direction::kSender; }
std::string GetMimeType() const override { return codec_mime_type_; }
rtc::ArrayView<const uint32_t> GetContributingSources() const override {
return {};
@ -100,6 +103,7 @@ class TransformableOutgoingAudioFrame
rtc::Buffer payload_;
absl::optional<uint64_t> absolute_capture_timestamp_ms_;
uint32_t ssrc_;
std::string codec_mime_type_;
};
} // namespace
@ -131,11 +135,12 @@ void ChannelSendFrameTransformerDelegate::Transform(
const uint8_t* payload_data,
size_t payload_size,
int64_t absolute_capture_timestamp_ms,
uint32_t ssrc) {
uint32_t ssrc,
const std::string& codec_mimetype) {
frame_transformer_->Transform(
std::make_unique<TransformableOutgoingAudioFrame>(
frame_type, payload_type, rtp_timestamp, payload_data, payload_size,
absolute_capture_timestamp_ms, ssrc));
absolute_capture_timestamp_ms, ssrc, codec_mimetype));
}
void ChannelSendFrameTransformerDelegate::OnTransformedFrame(
@ -173,7 +178,8 @@ std::unique_ptr<TransformableAudioFrameInterface> CloneSenderAudioFrame(
InterfaceFrameTypeToInternalFrameType(original->Type()),
original->GetPayloadType(), original->GetTimestamp(),
original->GetData().data(), original->GetData().size(),
original->AbsoluteCaptureTimestamp(), original->GetSsrc());
original->AbsoluteCaptureTimestamp(), original->GetSsrc(),
original->GetMimeType());
}
} // namespace webrtc

4
audio/channel_send_frame_transformer_delegate.h
View file

@ -12,6 +12,7 @@
#define AUDIO_CHANNEL_SEND_FRAME_TRANSFORMER_DELEGATE_H_
#include <memory>
#include <string>
#include "api/frame_transformer_interface.h"
#include "api/sequence_checker.h"
@ -57,7 +58,8 @@ class ChannelSendFrameTransformerDelegate : public TransformedFrameCallback {
const uint8_t* payload_data,
size_t payload_size,
int64_t absolute_capture_timestamp_ms,
uint32_t ssrc);
uint32_t ssrc,
const std::string& codec_mime_type);
// Implements TransformedFrameCallback. Can be called on any thread.
void OnTransformedFrame(

4
audio/channel_send_frame_transformer_delegate_unittest.cc
View file

@ -115,7 +115,7 @@ TEST(ChannelSendFrameTransformerDelegateTest,
callback->OnTransformedFrame(std::move(frame));
});
delegate->Transform(AudioFrameType::kEmptyFrame, 0, 0, data, sizeof(data), 0,
0);
/*ssrc=*/0, /*mimeType=*/"audio/opus");
channel_queue.WaitForPreviouslyPostedTasks();
}
@ -145,7 +145,7 @@ TEST(ChannelSendFrameTransformerDelegateTest,
callback->OnTransformedFrame(CreateMockReceiverFrame());
});
delegate->Transform(AudioFrameType::kEmptyFrame, 0, 0, data, sizeof(data), 0,
0);
/*ssrc=*/0, /*mimeType=*/"audio/opus");
channel_queue.WaitForPreviouslyPostedTasks();
}

7
audio/channel_send_unittest.cc
View file

@ -66,9 +66,10 @@ class ChannelSendTest : public ::testing::Test {
&transport_, nullptr, &event_log_, nullptr, crypto_options_, false,
kRtcpIntervalMs, kSsrc, nullptr, &transport_controller_, field_trials_);
encoder_factory_ = CreateBuiltinAudioEncoderFactory();
std::unique_ptr<AudioEncoder> encoder = encoder_factory_->MakeAudioEncoder(
kPayloadType, SdpAudioFormat("opus", kRtpRateHz, 2), {});
channel_->SetEncoder(kPayloadType, std::move(encoder));
SdpAudioFormat opus = SdpAudioFormat("opus", kRtpRateHz, 2);
std::unique_ptr<AudioEncoder> encoder =
encoder_factory_->MakeAudioEncoder(kPayloadType, opus, {});
channel_->SetEncoder(kPayloadType, opus, std::move(encoder));
transport_controller_.EnsureStarted();
channel_->RegisterSenderCongestionControlObjects(&transport_controller_);
ON_CALL(transport_, SendRtcp).WillByDefault(Return(true));

4
audio/mock_voe_channel_proxy.h
View file

@ -113,7 +113,9 @@ class MockChannelSend : public voe::ChannelSendInterface {
public:
MOCK_METHOD(void,
SetEncoder,
(int payload_type, std::unique_ptr<AudioEncoder> encoder),
(int payload_type,
const SdpAudioFormat& encoder_format,
std::unique_ptr<AudioEncoder> encoder),
(override));
MOCK_METHOD(
void,