From 479a3c0f92b957a9ba1c242e7fe6e4d37b49ce32 Mon Sep 17 00:00:00 2001 From: Mirta Dvornicic Date: Tue, 4 Jun 2019 15:38:50 +0200 Subject: [PATCH] Add support for enabling and negotiating raw RTP packetization. Raw RTP packetization is done using the existing RtpPacketizerGeneric without adding the generic payload header. It is intended to be used together with generic frame descriptor RTP header extension. Bug: webrtc:10625 Change-Id: I2e3d0a766e4933ddc4ad4abc1449b9b91ba6cd35 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/138061 Commit-Queue: Mirta Dvornicic Reviewed-by: Harald Alvestrand Reviewed-by: Magnus Jedvert Reviewed-by: Karl Wiberg Reviewed-by: Rasmus Brandt Cr-Commit-Position: refs/heads/master@{#28154} --- api/peer_connection_interface.h | 4 + media/base/codec.cc | 11 +- media/base/codec.h | 8 + media/base/codec_unittest.cc | 32 ++++ media/base/media_constants.cc | 2 + media/base/media_constants.h | 2 + media/engine/webrtc_video_engine.cc | 6 + media/engine/webrtc_video_engine_unittest.cc | 57 +++++++ pc/channel.cc | 57 ++++++- pc/channel_unittest.cc | 147 +++++++++++++++++++ pc/media_session.cc | 30 ++++ pc/media_session.h | 1 + pc/peer_connection.cc | 2 + pc/peer_connection_media_unittest.cc | 101 +++++++++++++ pc/webrtc_sdp.cc | 75 ++++++++++ pc/webrtc_sdp_unittest.cc | 64 ++++++++ sdk/media_constraints.cc | 9 ++ sdk/media_constraints.h | 4 + 18 files changed, 609 insertions(+), 3 deletions(-) diff --git a/api/peer_connection_interface.h b/api/peer_connection_interface.h index a9d9785195..a23b25dbc9 100644 --- a/api/peer_connection_interface.h +++ b/api/peer_connection_interface.h @@ -682,6 +682,10 @@ class RTC_EXPORT PeerConnectionInterface : public rtc::RefCountInterface { // confused with RTCP mux (multiplexing RTP and RTCP together). bool use_rtp_mux = true; + // If true, "a=packetization: raw" attribute will be offered + // in the SDP for all video payload and accepted in the answer if offered. + bool raw_packetization_for_video = false; + // This will apply to all video tracks with a Plan B SDP offer/answer. int num_simulcast_layers = 1; diff --git a/media/base/codec.cc b/media/base/codec.cc index e95517dd3a..168e7a7829 100644 --- a/media/base/codec.cc +++ b/media/base/codec.cc @@ -294,7 +294,7 @@ void VideoCodec::SetDefaultParameters() { } bool VideoCodec::operator==(const VideoCodec& c) const { - return Codec::operator==(c); + return Codec::operator==(c) && packetization == c.packetization; } bool VideoCodec::Matches(const VideoCodec& other) const { @@ -302,6 +302,15 @@ bool VideoCodec::Matches(const VideoCodec& other) const { IsSameCodecSpecific(name, params, other.name, other.params); } +absl::optional VideoCodec::IntersectPacketization( + const VideoCodec& local_codec, + const VideoCodec& remote_codec) { + if (local_codec.packetization == remote_codec.packetization) { + return local_codec.packetization; + } + return absl::nullopt; +} + VideoCodec VideoCodec::CreateRtxCodec(int rtx_payload_type, int associated_payload_type) { VideoCodec rtx_codec(rtx_payload_type, kRtxCodecName); diff --git a/media/base/codec.h b/media/base/codec.h index b1a1f07238..f327199484 100644 --- a/media/base/codec.h +++ b/media/base/codec.h @@ -16,6 +16,7 @@ #include #include +#include "absl/types/optional.h" #include "api/rtp_parameters.h" #include "api/video_codecs/sdp_video_format.h" #include "media/base/media_constants.h" @@ -144,6 +145,8 @@ struct AudioCodec : public Codec { }; struct RTC_EXPORT VideoCodec : public Codec { + absl::optional packetization; + // Creates a codec with the given parameters. VideoCodec(int id, const std::string& name); // Creates a codec with the given name and empty id. @@ -171,6 +174,11 @@ struct RTC_EXPORT VideoCodec : public Codec { bool operator!=(const VideoCodec& c) const { return !(*this == c); } + // Return packetization which both |local_codec| and |remote_codec| support. + static absl::optional IntersectPacketization( + const VideoCodec& local_codec, + const VideoCodec& remote_codec); + static VideoCodec CreateRtxCodec(int rtx_payload_type, int associated_payload_type); diff --git a/media/base/codec_unittest.cc b/media/base/codec_unittest.cc index b3cda813b2..bf0addb5a3 100644 --- a/media/base/codec_unittest.cc +++ b/media/base/codec_unittest.cc @@ -174,6 +174,29 @@ TEST(CodecTest, TestVideoCodecOperators) { EXPECT_TRUE(c13 == c10); } +TEST(CodecTest, TestVideoCodecIntersectPacketization) { + VideoCodec c1; + c1.packetization = "raw"; + VideoCodec c2; + c2.packetization = "raw"; + VideoCodec c3; + + EXPECT_EQ(VideoCodec::IntersectPacketization(c1, c2), "raw"); + EXPECT_EQ(VideoCodec::IntersectPacketization(c1, c3), absl::nullopt); +} + +TEST(CodecTest, TestVideoCodecEqualsWithDifferentPacketization) { + VideoCodec c0(100, cricket::kVp8CodecName); + VideoCodec c1(100, cricket::kVp8CodecName); + VideoCodec c2(100, cricket::kVp8CodecName); + c2.packetization = "raw"; + + EXPECT_EQ(c0, c1); + EXPECT_NE(c0, c2); + EXPECT_NE(c2, c0); + EXPECT_EQ(c2, c2); +} + TEST(CodecTest, TestVideoCodecMatches) { // Test a codec with a static payload type. VideoCodec c0(95, "V"); @@ -190,6 +213,15 @@ TEST(CodecTest, TestVideoCodecMatches) { EXPECT_FALSE(c1.Matches(VideoCodec(95, "V"))); } +TEST(CodecTest, TestVideoCodecMatchesWithDifferentPacketization) { + VideoCodec c0(100, cricket::kVp8CodecName); + VideoCodec c1(101, cricket::kVp8CodecName); + c1.packetization = "raw"; + + EXPECT_TRUE(c0.Matches(c1)); + EXPECT_TRUE(c1.Matches(c0)); +} + // VP9 codecs compare profile information. TEST(CodecTest, TestVP9CodecMatches) { const char kProfile0[] = "0"; diff --git a/media/base/media_constants.cc b/media/base/media_constants.cc index 3b4d2f27f8..5bd4b754d2 100644 --- a/media/base/media_constants.cc +++ b/media/base/media_constants.cc @@ -77,6 +77,8 @@ const int kPreferredSPropStereo = 0; const int kPreferredStereo = 0; const int kPreferredUseInbandFec = 0; +const char kPacketizationParamRaw[] = "raw"; + const char kRtcpFbParamLntf[] = "goog-lntf"; const char kRtcpFbParamNack[] = "nack"; const char kRtcpFbNackParamPli[] = "pli"; diff --git a/media/base/media_constants.h b/media/base/media_constants.h index a796474dc3..136e9f19a0 100644 --- a/media/base/media_constants.h +++ b/media/base/media_constants.h @@ -91,6 +91,8 @@ extern const int kPreferredSPropStereo; extern const int kPreferredStereo; extern const int kPreferredUseInbandFec; +extern const char kPacketizationParamRaw[]; + // rtcp-fb message in its first experimental stages. Documentation pending. extern const char kRtcpFbParamLntf[]; // rtcp-fb messages according to RFC 4585 diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index 4219c438ef..ae366d66cd 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -1877,6 +1877,8 @@ void WebRtcVideoChannel::WebRtcVideoSendStream::SetCodec( parameters_.config.rtp.payload_name = codec_settings.codec.name; parameters_.config.rtp.payload_type = codec_settings.codec.id; + parameters_.config.rtp.raw_payload = + codec_settings.codec.packetization == kPacketizationParamRaw; parameters_.config.rtp.ulpfec = codec_settings.ulpfec; parameters_.config.rtp.flexfec.payload_type = codec_settings.flexfec_payload_type; @@ -2463,6 +2465,7 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::ConfigureCodecs( RTC_DCHECK(!recv_codecs.empty()); config_.decoders.clear(); config_.rtp.rtx_associated_payload_types.clear(); + config_.rtp.raw_payload_types.clear(); for (const auto& recv_codec : recv_codecs) { webrtc::SdpVideoFormat video_format(recv_codec.codec.name, recv_codec.codec.params); @@ -2476,6 +2479,9 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::ConfigureCodecs( config_.decoders.push_back(decoder); config_.rtp.rtx_associated_payload_types[recv_codec.rtx_payload_type] = recv_codec.codec.id; + if (recv_codec.codec.packetization == kPacketizationParamRaw) { + config_.rtp.raw_payload_types.insert(recv_codec.codec.id); + } } const auto& codec = recv_codecs.front(); diff --git a/media/engine/webrtc_video_engine_unittest.cc b/media/engine/webrtc_video_engine_unittest.cc index f2b6e373ab..dc94f5dd89 100644 --- a/media/engine/webrtc_video_engine_unittest.cc +++ b/media/engine/webrtc_video_engine_unittest.cc @@ -3617,6 +3617,27 @@ TEST_F(WebRtcVideoChannelTest, SetDefaultSendCodecs) { // TODO(juberti): Check RTCP, PLI, TMMBR. } +TEST_F(WebRtcVideoChannelTest, SetSendCodecsWithoutPacketization) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + + FakeVideoSendStream* stream = AddSendStream(); + const webrtc::VideoSendStream::Config config = stream->GetConfig().Copy(); + EXPECT_FALSE(config.rtp.raw_payload); +} + +TEST_F(WebRtcVideoChannelTest, SetSendCodecsWithPacketization) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.back().packetization = kPacketizationParamRaw; + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + + FakeVideoSendStream* stream = AddSendStream(); + const webrtc::VideoSendStream::Config config = stream->GetConfig().Copy(); + EXPECT_TRUE(config.rtp.raw_payload); +} + // The following four tests ensures that FlexFEC is not activated by default // when the field trials are not enabled. // TODO(brandtr): Remove or update these tests when FlexFEC _is_ enabled by @@ -4443,6 +4464,42 @@ TEST_F(WebRtcVideoChannelTest, SetRecvCodecsWithRtx) { "rejected."; } +TEST_F(WebRtcVideoChannelTest, SetRecvCodecsWithPacketization) { + cricket::VideoCodec vp8_codec = GetEngineCodec("VP8"); + vp8_codec.packetization = kPacketizationParamRaw; + + cricket::VideoRecvParameters parameters; + parameters.codecs = {vp8_codec, GetEngineCodec("VP9")}; + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + + const cricket::StreamParams params = + cricket::StreamParams::CreateLegacy(kSsrcs1[0]); + AddRecvStream(params); + ASSERT_THAT(fake_call_->GetVideoReceiveStreams(), testing::SizeIs(1)); + + const webrtc::VideoReceiveStream::Config& config = + fake_call_->GetVideoReceiveStreams()[0]->GetConfig(); + ASSERT_THAT(config.rtp.raw_payload_types, testing::SizeIs(1)); + EXPECT_EQ(config.rtp.raw_payload_types.count(vp8_codec.id), 1U); +} + +TEST_F(WebRtcVideoChannelTest, SetRecvCodecsWithPacketizationRecreatesStream) { + cricket::VideoRecvParameters parameters; + parameters.codecs = {GetEngineCodec("VP8"), GetEngineCodec("VP9")}; + parameters.codecs.back().packetization = kPacketizationParamRaw; + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + + const cricket::StreamParams params = + cricket::StreamParams::CreateLegacy(kSsrcs1[0]); + AddRecvStream(params); + ASSERT_THAT(fake_call_->GetVideoReceiveStreams(), testing::SizeIs(1)); + EXPECT_EQ(fake_call_->GetNumCreatedReceiveStreams(), 1); + + parameters.codecs.back().packetization.reset(); + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + EXPECT_EQ(fake_call_->GetNumCreatedReceiveStreams(), 2); +} + TEST_F(WebRtcVideoChannelTest, SetRecvCodecsWithChangedRtxPayloadType) { const int kUnusedPayloadType1 = 126; const int kUnusedPayloadType2 = 127; diff --git a/pc/channel.cc b/pc/channel.cc index 13341e3e1b..54839631cc 100644 --- a/pc/channel.cc +++ b/pc/channel.cc @@ -1021,6 +1021,26 @@ bool VideoChannel::SetLocalContent_w(const MediaContentDescription* content, VideoRecvParameters recv_params = last_recv_params_; RtpParametersFromMediaDescription(video, rtp_header_extensions, &recv_params); + + VideoSendParameters send_params = last_send_params_; + bool needs_send_params_update = false; + if (type == SdpType::kAnswer || type == SdpType::kPrAnswer) { + for (auto& send_codec : send_params.codecs) { + auto* recv_codec = FindMatchingCodec(recv_params.codecs, send_codec); + if (recv_codec) { + if (!recv_codec->packetization && send_codec.packetization) { + send_codec.packetization.reset(); + needs_send_params_update = true; + } else if (recv_codec->packetization != send_codec.packetization) { + SafeSetError( + "Failed to set local answer due to invalid codec packetization.", + error_desc); + return false; + } + } + } + } + if (!media_channel()->SetRecvParameters(recv_params)) { SafeSetError("Failed to set local video description recv parameters.", error_desc); @@ -1037,6 +1057,14 @@ bool VideoChannel::SetLocalContent_w(const MediaContentDescription* content, last_recv_params_ = recv_params; + if (needs_send_params_update) { + if (!media_channel()->SetSendParameters(send_params)) { + SafeSetError("Failed to set send parameters.", error_desc); + return false; + } + last_send_params_ = send_params; + } + // TODO(pthatcher): Move local streams into VideoSendParameters, and // only give it to the media channel once we have a remote // description too (without a remote description, we won't be able @@ -1077,15 +1105,40 @@ bool VideoChannel::SetRemoteContent_w(const MediaContentDescription* content, } send_params.mid = content_name(); - bool parameters_applied = media_channel()->SetSendParameters(send_params); + VideoRecvParameters recv_params = last_recv_params_; + bool needs_recv_params_update = false; + if (type == SdpType::kAnswer || type == SdpType::kPrAnswer) { + for (auto& recv_codec : recv_params.codecs) { + auto* send_codec = FindMatchingCodec(send_params.codecs, recv_codec); + if (send_codec) { + if (!send_codec->packetization && recv_codec.packetization) { + recv_codec.packetization.reset(); + needs_recv_params_update = true; + } else if (send_codec->packetization != recv_codec.packetization) { + SafeSetError( + "Failed to set remote answer due to invalid codec packetization.", + error_desc); + return false; + } + } + } + } - if (!parameters_applied) { + if (!media_channel()->SetSendParameters(send_params)) { SafeSetError("Failed to set remote video description send parameters.", error_desc); return false; } last_send_params_ = send_params; + if (needs_recv_params_update) { + if (!media_channel()->SetRecvParameters(recv_params)) { + SafeSetError("Failed to set recv parameters.", error_desc); + return false; + } + last_recv_params_ = recv_params; + } + // TODO(pthatcher): Move remote streams into VideoRecvParameters, // and only give it to the media channel once we have a local // description too (without a local description, we won't be able to diff --git a/pc/channel_unittest.cc b/pc/channel_unittest.cc index e4252b2e8b..db0e8a8d02 100644 --- a/pc/channel_unittest.cc +++ b/pc/channel_unittest.cc @@ -37,6 +37,7 @@ #include "rtc_base/checks.h" #include "rtc_base/rtc_certificate.h" #include "rtc_base/ssl_identity.h" +#include "test/gmock.h" #include "test/gtest.h" using cricket::DtlsTransportInternal; @@ -2121,6 +2122,152 @@ TEST_F(VideoChannelSingleThreadTest, UpdateLocalStreamsWithSimulcast) { Base::TestUpdateLocalStreamsWithSimulcast(); } +TEST_F(VideoChannelSingleThreadTest, TestSetLocalOfferWithPacketization) { + const cricket::VideoCodec kVp8Codec(97, "VP8"); + cricket::VideoCodec vp9_codec(98, "VP9"); + vp9_codec.packetization = cricket::kPacketizationParamRaw; + cricket::VideoContentDescription video; + video.set_codecs({kVp8Codec, vp9_codec}); + + CreateChannels(0, 0); + + EXPECT_TRUE(channel1_->SetLocalContent(&video, SdpType::kOffer, NULL)); + EXPECT_THAT(media_channel1_->send_codecs(), testing::IsEmpty()); + ASSERT_THAT(media_channel1_->recv_codecs(), testing::SizeIs(2)); + EXPECT_TRUE(media_channel1_->recv_codecs()[0].Matches(kVp8Codec)); + EXPECT_EQ(media_channel1_->recv_codecs()[0].packetization, absl::nullopt); + EXPECT_TRUE(media_channel1_->recv_codecs()[1].Matches(vp9_codec)); + EXPECT_EQ(media_channel1_->recv_codecs()[1].packetization, + cricket::kPacketizationParamRaw); +} + +TEST_F(VideoChannelSingleThreadTest, TestSetRemoteOfferWithPacketization) { + const cricket::VideoCodec kVp8Codec(97, "VP8"); + cricket::VideoCodec vp9_codec(98, "VP9"); + vp9_codec.packetization = cricket::kPacketizationParamRaw; + cricket::VideoContentDescription video; + video.set_codecs({kVp8Codec, vp9_codec}); + + CreateChannels(0, 0); + + EXPECT_TRUE(channel1_->SetRemoteContent(&video, SdpType::kOffer, NULL)); + EXPECT_THAT(media_channel1_->recv_codecs(), testing::IsEmpty()); + ASSERT_THAT(media_channel1_->send_codecs(), testing::SizeIs(2)); + EXPECT_TRUE(media_channel1_->send_codecs()[0].Matches(kVp8Codec)); + EXPECT_EQ(media_channel1_->send_codecs()[0].packetization, absl::nullopt); + EXPECT_TRUE(media_channel1_->send_codecs()[1].Matches(vp9_codec)); + EXPECT_EQ(media_channel1_->send_codecs()[1].packetization, + cricket::kPacketizationParamRaw); +} + +TEST_F(VideoChannelSingleThreadTest, TestSetAnswerWithPacketization) { + const cricket::VideoCodec kVp8Codec(97, "VP8"); + cricket::VideoCodec vp9_codec(98, "VP9"); + vp9_codec.packetization = cricket::kPacketizationParamRaw; + cricket::VideoContentDescription video; + video.set_codecs({kVp8Codec, vp9_codec}); + + CreateChannels(0, 0); + + EXPECT_TRUE(channel1_->SetLocalContent(&video, SdpType::kOffer, NULL)); + EXPECT_TRUE(channel1_->SetRemoteContent(&video, SdpType::kAnswer, NULL)); + ASSERT_THAT(media_channel1_->recv_codecs(), testing::SizeIs(2)); + EXPECT_TRUE(media_channel1_->recv_codecs()[0].Matches(kVp8Codec)); + EXPECT_EQ(media_channel1_->recv_codecs()[0].packetization, absl::nullopt); + EXPECT_TRUE(media_channel1_->recv_codecs()[1].Matches(vp9_codec)); + EXPECT_EQ(media_channel1_->recv_codecs()[1].packetization, + cricket::kPacketizationParamRaw); + EXPECT_THAT(media_channel1_->send_codecs(), testing::SizeIs(2)); + EXPECT_TRUE(media_channel1_->send_codecs()[0].Matches(kVp8Codec)); + EXPECT_EQ(media_channel1_->send_codecs()[0].packetization, absl::nullopt); + EXPECT_TRUE(media_channel1_->send_codecs()[1].Matches(vp9_codec)); + EXPECT_EQ(media_channel1_->send_codecs()[1].packetization, + cricket::kPacketizationParamRaw); +} + +TEST_F(VideoChannelSingleThreadTest, TestSetLocalAnswerWithoutPacketization) { + const cricket::VideoCodec kLocalCodec(98, "VP8"); + cricket::VideoCodec remote_codec(99, "VP8"); + remote_codec.packetization = cricket::kPacketizationParamRaw; + cricket::VideoContentDescription local_video; + local_video.set_codecs({kLocalCodec}); + cricket::VideoContentDescription remote_video; + remote_video.set_codecs({remote_codec}); + + CreateChannels(0, 0); + + EXPECT_TRUE( + channel1_->SetRemoteContent(&remote_video, SdpType::kOffer, NULL)); + EXPECT_TRUE(channel1_->SetLocalContent(&local_video, SdpType::kAnswer, NULL)); + ASSERT_THAT(media_channel1_->recv_codecs(), testing::SizeIs(1)); + EXPECT_EQ(media_channel1_->recv_codecs()[0].packetization, absl::nullopt); + ASSERT_THAT(media_channel1_->send_codecs(), testing::SizeIs(1)); + EXPECT_EQ(media_channel1_->send_codecs()[0].packetization, absl::nullopt); +} + +TEST_F(VideoChannelSingleThreadTest, TestSetRemoteAnswerWithoutPacketization) { + cricket::VideoCodec local_codec(98, "VP8"); + local_codec.packetization = cricket::kPacketizationParamRaw; + const cricket::VideoCodec kRemoteCodec(99, "VP8"); + cricket::VideoContentDescription local_video; + local_video.set_codecs({local_codec}); + cricket::VideoContentDescription remote_video; + remote_video.set_codecs({kRemoteCodec}); + + CreateChannels(0, 0); + + EXPECT_TRUE(channel1_->SetLocalContent(&local_video, SdpType::kOffer, NULL)); + EXPECT_TRUE( + channel1_->SetRemoteContent(&remote_video, SdpType::kAnswer, NULL)); + ASSERT_THAT(media_channel1_->recv_codecs(), testing::SizeIs(1)); + EXPECT_EQ(media_channel1_->recv_codecs()[0].packetization, absl::nullopt); + ASSERT_THAT(media_channel1_->send_codecs(), testing::SizeIs(1)); + EXPECT_EQ(media_channel1_->send_codecs()[0].packetization, absl::nullopt); +} + +TEST_F(VideoChannelSingleThreadTest, + TestSetRemoteAnswerWithInvalidPacketization) { + cricket::VideoCodec local_codec(98, "VP8"); + local_codec.packetization = cricket::kPacketizationParamRaw; + cricket::VideoCodec remote_codec(99, "VP8"); + remote_codec.packetization = "unknownpacketizationattributevalue"; + cricket::VideoContentDescription local_video; + local_video.set_codecs({local_codec}); + cricket::VideoContentDescription remote_video; + remote_video.set_codecs({remote_codec}); + + CreateChannels(0, 0); + + EXPECT_TRUE(channel1_->SetLocalContent(&local_video, SdpType::kOffer, NULL)); + EXPECT_FALSE( + channel1_->SetRemoteContent(&remote_video, SdpType::kAnswer, NULL)); + ASSERT_THAT(media_channel1_->recv_codecs(), testing::SizeIs(1)); + EXPECT_EQ(media_channel1_->recv_codecs()[0].packetization, + cricket::kPacketizationParamRaw); + EXPECT_THAT(media_channel1_->send_codecs(), testing::IsEmpty()); +} + +TEST_F(VideoChannelSingleThreadTest, + TestSetLocalAnswerWithInvalidPacketization) { + cricket::VideoCodec local_codec(98, "VP8"); + local_codec.packetization = cricket::kPacketizationParamRaw; + const cricket::VideoCodec kRemoteCodec(99, "VP8"); + cricket::VideoContentDescription local_video; + local_video.set_codecs({local_codec}); + cricket::VideoContentDescription remote_video; + remote_video.set_codecs({kRemoteCodec}); + + CreateChannels(0, 0); + + EXPECT_TRUE( + channel1_->SetRemoteContent(&remote_video, SdpType::kOffer, NULL)); + EXPECT_FALSE( + channel1_->SetLocalContent(&local_video, SdpType::kAnswer, NULL)); + EXPECT_THAT(media_channel1_->recv_codecs(), testing::IsEmpty()); + ASSERT_THAT(media_channel1_->send_codecs(), testing::SizeIs(1)); + EXPECT_EQ(media_channel1_->send_codecs()[0].packetization, absl::nullopt); +} + // VideoChannelDoubleThreadTest TEST_F(VideoChannelDoubleThreadTest, TestInit) { Base::TestInit(); diff --git a/pc/media_session.cc b/pc/media_session.cc index 7cc4b95f14..fe11d44702 100644 --- a/pc/media_session.cc +++ b/pc/media_session.cc @@ -786,6 +786,19 @@ static bool ReferencedCodecsMatch(const std::vector& codecs1, return codec1 != nullptr && codec2 != nullptr && codec1->Matches(*codec2); } +template +static void NegotiatePacketization(const C& local_codec, + const C& remote_codec, + C* negotiated_codec) {} + +template <> +void NegotiatePacketization(const VideoCodec& local_codec, + const VideoCodec& remote_codec, + VideoCodec* negotiated_codec) { + negotiated_codec->packetization = + VideoCodec::IntersectPacketization(local_codec, remote_codec); +} + template static void NegotiateCodecs(const std::vector& local_codecs, const std::vector& offered_codecs, @@ -797,6 +810,7 @@ static void NegotiateCodecs(const std::vector& local_codecs, // local codecs, in case the remote offer contains duplicate codecs. if (FindMatchingCodec(local_codecs, offered_codecs, ours, &theirs)) { C negotiated = ours; + NegotiatePacketization(ours, theirs, &negotiated); negotiated.IntersectFeedbackParams(theirs); if (IsRtxCodec(negotiated)) { const auto apt_it = @@ -2187,6 +2201,14 @@ bool MediaSessionDescriptionFactory::AddVideoContentForOffer( } } + if (session_options.raw_packetization_for_video) { + for (VideoCodec& codec : filtered_codecs) { + if (codec.GetCodecType() == VideoCodec::CODEC_VIDEO) { + codec.packetization = kPacketizationParamRaw; + } + } + } + if (!CreateMediaContentOffer(media_description_options, session_options, filtered_codecs, sdes_policy, GetCryptos(current_content), crypto_suites, @@ -2504,6 +2526,14 @@ bool MediaSessionDescriptionFactory::AddVideoContentForAnswer( } } + if (session_options.raw_packetization_for_video) { + for (VideoCodec& codec : filtered_codecs) { + if (codec.GetCodecType() == VideoCodec::CODEC_VIDEO) { + codec.packetization = kPacketizationParamRaw; + } + } + } + bool bundle_enabled = offer_description->HasGroup(GROUP_TYPE_BUNDLE) && session_options.bundle_enabled; diff --git a/pc/media_session.h b/pc/media_session.h index 8ba6293b66..b154a1816d 100644 --- a/pc/media_session.h +++ b/pc/media_session.h @@ -106,6 +106,7 @@ struct MediaSessionOptions { bool rtcp_mux_enabled = true; bool bundle_enabled = false; bool offer_extmap_allow_mixed = false; + bool raw_packetization_for_video = false; std::string rtcp_cname = kDefaultRtcpCname; webrtc::CryptoOptions crypto_options; // List of media description options in the same order that the media diff --git a/pc/peer_connection.cc b/pc/peer_connection.cc index 82eb425d4d..fde0433278 100644 --- a/pc/peer_connection.cc +++ b/pc/peer_connection.cc @@ -858,6 +858,8 @@ void ExtractSharedMediaSessionOptions( cricket::MediaSessionOptions* session_options) { session_options->vad_enabled = rtc_options.voice_activity_detection; session_options->bundle_enabled = rtc_options.use_rtp_mux; + session_options->raw_packetization_for_video = + rtc_options.raw_packetization_for_video; } PeerConnection::PeerConnection(PeerConnectionFactory* factory, diff --git a/pc/peer_connection_media_unittest.cc b/pc/peer_connection_media_unittest.cc index 1ff9f07198..ac2563d5fd 100644 --- a/pc/peer_connection_media_unittest.cc +++ b/pc/peer_connection_media_unittest.cc @@ -15,6 +15,7 @@ #include #include "absl/algorithm/container.h" +#include "absl/types/optional.h" #include "api/call/call_factory_interface.h" #include "api/test/fake_media_transport.h" #include "logging/rtc_event_log/rtc_event_log_factory.h" @@ -516,6 +517,106 @@ TEST_P(PeerConnectionMediaTest, EXPECT_FALSE(video_content->rejected); } +// Test that raw packetization is not set in the offer by default. +TEST_P(PeerConnectionMediaTest, RawPacketizationNotSetInOffer) { + std::vector fake_codecs; + fake_codecs.push_back(cricket::VideoCodec(111, cricket::kVp8CodecName)); + fake_codecs.push_back(cricket::VideoCodec(112, cricket::kRtxCodecName)); + fake_codecs.back().params[cricket::kCodecParamAssociatedPayloadType] = "111"; + fake_codecs.push_back(cricket::VideoCodec(113, cricket::kVp9CodecName)); + fake_codecs.push_back(cricket::VideoCodec(114, cricket::kH264CodecName)); + fake_codecs.push_back(cricket::VideoCodec(115, "HEVC")); + auto caller_fake_engine = absl::make_unique(); + caller_fake_engine->SetVideoCodecs(fake_codecs); + + auto caller = CreatePeerConnectionWithVideo(std::move(caller_fake_engine)); + auto offer = caller->CreateOfferAndSetAsLocal(); + auto* offer_description = + cricket::GetFirstVideoContentDescription(offer->description()); + for (const auto& codec : offer_description->codecs()) { + EXPECT_EQ(codec.packetization, absl::nullopt); + } +} + +// Test that raw packetization is set in the offer and answer for all +// video payload when raw_packetization_for_video is true. +TEST_P(PeerConnectionMediaTest, RawPacketizationSetInOfferAndAnswer) { + std::vector fake_codecs; + fake_codecs.push_back(cricket::VideoCodec(111, cricket::kVp8CodecName)); + fake_codecs.push_back(cricket::VideoCodec(112, cricket::kRtxCodecName)); + fake_codecs.back().params[cricket::kCodecParamAssociatedPayloadType] = "111"; + fake_codecs.push_back(cricket::VideoCodec(113, cricket::kVp9CodecName)); + fake_codecs.push_back(cricket::VideoCodec(114, cricket::kH264CodecName)); + fake_codecs.push_back(cricket::VideoCodec(115, "HEVC")); + auto caller_fake_engine = absl::make_unique(); + caller_fake_engine->SetVideoCodecs(fake_codecs); + auto callee_fake_engine = absl::make_unique(); + callee_fake_engine->SetVideoCodecs(fake_codecs); + + RTCOfferAnswerOptions options; + options.raw_packetization_for_video = true; + + auto caller = CreatePeerConnectionWithVideo(std::move(caller_fake_engine)); + auto offer = caller->CreateOfferAndSetAsLocal(options); + auto* offer_description = + cricket::GetFirstVideoContentDescription(offer->description()); + for (const auto& codec : offer_description->codecs()) { + if (codec.GetCodecType() == cricket::VideoCodec::CODEC_VIDEO) { + EXPECT_EQ(codec.packetization, cricket::kPacketizationParamRaw); + } + } + + auto callee = CreatePeerConnectionWithVideo(std::move(callee_fake_engine)); + ASSERT_TRUE(callee->SetRemoteDescription(std::move(offer))); + auto answer = callee->CreateAnswerAndSetAsLocal(options); + auto* answer_description = + cricket::GetFirstVideoContentDescription(answer->description()); + for (const auto& codec : answer_description->codecs()) { + if (codec.GetCodecType() == cricket::VideoCodec::CODEC_VIDEO) { + EXPECT_EQ(codec.packetization, cricket::kPacketizationParamRaw); + } + } + + ASSERT_TRUE(caller->SetRemoteDescription(std::move(answer))); +} + +// Test that raw packetization is not set in the answer when +// raw_packetization_for_video is true if it was not set in the offer. +TEST_P(PeerConnectionMediaTest, + RawPacketizationNotSetInAnswerWhenNotSetInOffer) { + std::vector fake_codecs; + fake_codecs.push_back(cricket::VideoCodec(111, cricket::kVp8CodecName)); + fake_codecs.push_back(cricket::VideoCodec(112, cricket::kRtxCodecName)); + fake_codecs.back().params[cricket::kCodecParamAssociatedPayloadType] = "111"; + fake_codecs.push_back(cricket::VideoCodec(113, cricket::kVp9CodecName)); + fake_codecs.push_back(cricket::VideoCodec(114, cricket::kH264CodecName)); + fake_codecs.push_back(cricket::VideoCodec(115, "HEVC")); + auto caller_fake_engine = absl::make_unique(); + caller_fake_engine->SetVideoCodecs(fake_codecs); + auto callee_fake_engine = absl::make_unique(); + callee_fake_engine->SetVideoCodecs(fake_codecs); + + RTCOfferAnswerOptions caller_options; + caller_options.raw_packetization_for_video = false; + RTCOfferAnswerOptions callee_options; + callee_options.raw_packetization_for_video = true; + + auto caller = CreatePeerConnectionWithVideo(std::move(caller_fake_engine)); + auto offer = caller->CreateOfferAndSetAsLocal(caller_options); + + auto callee = CreatePeerConnectionWithVideo(std::move(callee_fake_engine)); + ASSERT_TRUE(callee->SetRemoteDescription(std::move(offer))); + auto answer = callee->CreateAnswerAndSetAsLocal(callee_options); + + auto* answer_description = + cricket::GetFirstVideoContentDescription(answer->description()); + for (const auto& codec : answer_description->codecs()) { + EXPECT_EQ(codec.packetization, absl::nullopt); + } + + ASSERT_TRUE(caller->SetRemoteDescription(std::move(answer))); +} + class PeerConnectionMediaOfferDirectionTest : public PeerConnectionMediaBaseTest, public ::testing::WithParamInterface< diff --git a/pc/webrtc_sdp.cc b/pc/webrtc_sdp.cc index af8a5277de..83bd3782bd 100644 --- a/pc/webrtc_sdp.cc +++ b/pc/webrtc_sdp.cc @@ -180,6 +180,7 @@ static const char kAttributeSimulcast[] = "simulcast"; // draft-ietf-mmusic-rid-15 // a=rid static const char kAttributeRid[] = "rid"; +static const char kAttributePacketization[] = "packetization"; // Experimental flags static const char kAttributeXGoogleFlag[] = "x-google-flag"; @@ -344,6 +345,10 @@ static bool ParseFmtpParam(const std::string& line, std::string* parameter, std::string* value, SdpParseError* error); +static bool ParsePacketizationAttribute(const std::string& line, + const cricket::MediaType media_type, + MediaContentDescription* media_desc, + SdpParseError* error); static bool ParseRtcpFbAttribute(const std::string& line, const cricket::MediaType media_type, MediaContentDescription* media_desc, @@ -1744,6 +1749,14 @@ void WriteFmtpHeader(int payload_type, rtc::StringBuilder* os) { *os << kSdpDelimiterColon << payload_type; } +void WritePacketizationHeader(int payload_type, rtc::StringBuilder* os) { + // packetization header: a=packetization:|payload_type| + // Add a=packetization + InitAttrLine(kAttributePacketization, os); + // Add :|payload_type| + *os << kSdpDelimiterColon << payload_type; +} + void WriteRtcpFbHeader(int payload_type, rtc::StringBuilder* os) { // rtcp-fb header: a=rtcp-fb:|payload_type| // /> @@ -1819,6 +1832,17 @@ void AddFmtpLine(const T& codec, std::string* message) { return; } +template +void AddPacketizationLine(const T& codec, std::string* message) { + if (!codec.packetization) { + return; + } + rtc::StringBuilder os; + WritePacketizationHeader(codec.id, &os); + os << " " << *codec.packetization; + AddLine(os.str(), message); +} + template void AddRtcpFbLines(const T& codec, std::string* message) { for (const cricket::FeedbackParam& param : codec.feedback_params.params()) { @@ -1871,6 +1895,7 @@ void BuildRtpMap(const MediaContentDescription* media_desc, << cricket::kVideoCodecClockrate; AddLine(os.str(), message); } + AddPacketizationLine(codec, message); AddRtcpFbLines(codec, message); AddFmtpLine(codec, message); } @@ -2910,6 +2935,24 @@ void UpdateCodec(MediaContentDescription* content_desc, AddOrReplaceCodec(content_desc, new_codec); } +// Adds or updates existing video codec corresponding to |payload_type| +// according to |packetization|. +void UpdateVideoCodecPacketization(VideoContentDescription* video_desc, + int payload_type, + const std::string& packetization) { + if (packetization != cricket::kPacketizationParamRaw) { + // Ignore unsupported packetization attribute. + return; + } + + // Codec might already have been populated (from rtpmap). + cricket::VideoCodec codec = + GetCodecWithPayloadType(video_desc->codecs(), payload_type); + codec.packetization = packetization; + AddOrReplaceCodec(video_desc, + codec); +} + template bool PopWildcardCodec(std::vector* codecs, T* wildcard_codec) { for (auto iter = codecs->begin(); iter != codecs->end(); ++iter) { @@ -3166,6 +3209,10 @@ bool ParseContent(const std::string& message, if (!GetValue(line, kCodecParamMaxPTime, &maxptime_as_string, error)) { return false; } + } else if (HasAttribute(line, kAttributePacketization)) { + if (!ParsePacketizationAttribute(line, media_type, media_desc, error)) { + return false; + } } else if (HasAttribute(line, kAttributeRtcpFb)) { if (!ParseRtcpFbAttribute(line, media_type, media_desc, error)) { return false; @@ -3682,6 +3729,34 @@ bool ParseFmtpAttributes(const std::string& line, return true; } +bool ParsePacketizationAttribute(const std::string& line, + const cricket::MediaType media_type, + MediaContentDescription* media_desc, + SdpParseError* error) { + if (media_type != cricket::MEDIA_TYPE_VIDEO) { + return true; + } + std::vector packetization_fields; + rtc::split(line.c_str(), kSdpDelimiterSpaceChar, &packetization_fields); + if (packetization_fields.size() < 2) { + return ParseFailedGetValue(line, kAttributePacketization, error); + } + std::string payload_type_string; + if (!GetValue(packetization_fields[0], kAttributePacketization, + &payload_type_string, error)) { + return false; + } + int payload_type; + if (!GetPayloadTypeFromString(line, payload_type_string, &payload_type, + error)) { + return false; + } + std::string packetization = packetization_fields[1]; + UpdateVideoCodecPacketization(media_desc->as_video(), payload_type, + packetization); + return true; +} + bool ParseRtcpFbAttribute(const std::string& line, const cricket::MediaType media_type, MediaContentDescription* media_desc, diff --git a/pc/webrtc_sdp_unittest.cc b/pc/webrtc_sdp_unittest.cc index 4b7a6d9219..5fdf5f61ab 100644 --- a/pc/webrtc_sdp_unittest.cc +++ b/pc/webrtc_sdp_unittest.cc @@ -3418,6 +3418,54 @@ TEST_F(WebRtcSdpTest, DeserializeVideoFmtpWithSpace) { EXPECT_EQ(found->second, "40"); } +TEST_F(WebRtcSdpTest, DeserializePacketizationAttributeWithIllegalValue) { + JsepSessionDescription jdesc_output(kDummyType); + + const char kSdpWithPacketizationString[] = + "v=0\r\n" + "o=- 18446744069414584320 18446462598732840960 IN IP4 127.0.0.1\r\n" + "s=-\r\n" + "t=0 0\r\n" + "m=audio 9 RTP/SAVPF 111\r\n" + "a=rtpmap:111 opus/48000/2\r\n" + "a=packetization:111 unknownpacketizationattributeforaudio\r\n" + "m=video 3457 RTP/SAVPF 120 121 122\r\n" + "a=rtpmap:120 VP8/90000\r\n" + "a=packetization:120 raw\r\n" + "a=rtpmap:121 VP9/90000\r\n" + "a=rtpmap:122 H264/90000\r\n" + "a=packetization:122 unknownpacketizationattributevalue\r\n"; + + SdpParseError error; + EXPECT_TRUE(webrtc::SdpDeserialize(kSdpWithPacketizationString, &jdesc_output, + &error)); + + AudioContentDescription* acd = + GetFirstAudioContentDescription(jdesc_output.description()); + ASSERT_TRUE(acd); + ASSERT_THAT(acd->codecs(), testing::SizeIs(1)); + cricket::AudioCodec opus = acd->codecs()[0]; + EXPECT_EQ(opus.name, "opus"); + EXPECT_EQ(opus.id, 111); + + const VideoContentDescription* vcd = + GetFirstVideoContentDescription(jdesc_output.description()); + ASSERT_TRUE(vcd); + ASSERT_THAT(vcd->codecs(), testing::SizeIs(3)); + cricket::VideoCodec vp8 = vcd->codecs()[0]; + EXPECT_EQ(vp8.name, "VP8"); + EXPECT_EQ(vp8.id, 120); + EXPECT_EQ(vp8.packetization, "raw"); + cricket::VideoCodec vp9 = vcd->codecs()[1]; + EXPECT_EQ(vp9.name, "VP9"); + EXPECT_EQ(vp9.id, 121); + EXPECT_EQ(vp9.packetization, absl::nullopt); + cricket::VideoCodec h264 = vcd->codecs()[2]; + EXPECT_EQ(h264.name, "H264"); + EXPECT_EQ(h264.id, 122); + EXPECT_EQ(h264.packetization, absl::nullopt); +} + TEST_F(WebRtcSdpTest, SerializeAudioFmtpWithUnknownParameter) { AudioContentDescription* acd = GetFirstAudioContentDescription(&desc_); @@ -3486,6 +3534,22 @@ TEST_F(WebRtcSdpTest, SerializeVideoFmtp) { EXPECT_EQ(sdp_with_fmtp, message); } +TEST_F(WebRtcSdpTest, SerializeVideoPacketizationAttribute) { + VideoContentDescription* vcd = GetFirstVideoContentDescription(&desc_); + + cricket::VideoCodecs codecs = vcd->codecs(); + codecs[0].packetization = "raw"; + vcd->set_codecs(codecs); + + ASSERT_TRUE(jdesc_.Initialize(desc_.Clone(), jdesc_.session_id(), + jdesc_.session_version())); + std::string message = webrtc::SdpSerialize(jdesc_); + std::string sdp_with_packetization = kSdpFullString; + InjectAfter("a=rtpmap:120 VP8/90000\r\n", "a=packetization:120 raw\r\n", + &sdp_with_packetization); + EXPECT_EQ(sdp_with_packetization, message); +} + TEST_F(WebRtcSdpTest, DeserializeAndSerializeSdpWithIceLite) { // Deserialize the baseline description, making sure it's ICE full. JsepSessionDescription jdesc_with_icelite(kDummyType); diff --git a/sdk/media_constraints.cc b/sdk/media_constraints.cc index 7c5fe4a258..56d9e70ee2 100644 --- a/sdk/media_constraints.cc +++ b/sdk/media_constraints.cc @@ -134,6 +134,9 @@ const char MediaConstraints::kScreencastMinBitrate[] = // TODO(ronghuawu): Remove once cpu overuse detection is stable. const char MediaConstraints::kCpuOveruseDetection[] = "googCpuOveruseDetection"; +const char MediaConstraints::kRawPacketizationForVideoEnabled[] = + "googRawPacketizationForVideoEnabled"; + const char MediaConstraints::kNumSimulcastLayers[] = "googNumSimulcastLayers"; // Set |value| to the value associated with the first appearance of |key|, or @@ -262,6 +265,12 @@ bool CopyConstraintsIntoOfferAnswerOptions( offer_answer_options->ice_restart = value; } + if (FindConstraint(constraints, + MediaConstraints::kRawPacketizationForVideoEnabled, &value, + &mandatory_constraints_satisfied)) { + offer_answer_options->raw_packetization_for_video = value; + } + int layers; if (FindConstraint(constraints, MediaConstraints::kNumSimulcastLayers, &layers, &mandatory_constraints_satisfied)) { diff --git a/sdk/media_constraints.h b/sdk/media_constraints.h index 3a151417eb..f3e9853971 100644 --- a/sdk/media_constraints.h +++ b/sdk/media_constraints.h @@ -102,6 +102,10 @@ class MediaConstraints { static const char kScreencastMinBitrate[]; // googScreencastMinBitrate static const char kCpuOveruseDetection[]; // googCpuOveruseDetection + // Constraint to enable negotiating raw RTP packetization using attribute + // "a=packetization: raw" in the SDP for all video payload. + static const char kRawPacketizationForVideoEnabled[]; + // Specifies number of simulcast layers for all video tracks // with a Plan B offer/answer // (see RTCOfferAnswerOptions::num_simulcast_layers).