diff --git a/api/peer_connection_interface.h b/api/peer_connection_interface.h index a9d9785195..a23b25dbc9 100644 --- a/api/peer_connection_interface.h +++ b/api/peer_connection_interface.h @@ -682,6 +682,10 @@ class RTC_EXPORT PeerConnectionInterface : public rtc::RefCountInterface { // confused with RTCP mux (multiplexing RTP and RTCP together). bool use_rtp_mux = true; + // If true, "a=packetization: raw" attribute will be offered + // in the SDP for all video payload and accepted in the answer if offered. + bool raw_packetization_for_video = false; + // This will apply to all video tracks with a Plan B SDP offer/answer. int num_simulcast_layers = 1; diff --git a/media/base/codec.cc b/media/base/codec.cc index e95517dd3a..168e7a7829 100644 --- a/media/base/codec.cc +++ b/media/base/codec.cc @@ -294,7 +294,7 @@ void VideoCodec::SetDefaultParameters() { } bool VideoCodec::operator==(const VideoCodec& c) const { - return Codec::operator==(c); + return Codec::operator==(c) && packetization == c.packetization; } bool VideoCodec::Matches(const VideoCodec& other) const { @@ -302,6 +302,15 @@ bool VideoCodec::Matches(const VideoCodec& other) const { IsSameCodecSpecific(name, params, other.name, other.params); } +absl::optional VideoCodec::IntersectPacketization( + const VideoCodec& local_codec, + const VideoCodec& remote_codec) { + if (local_codec.packetization == remote_codec.packetization) { + return local_codec.packetization; + } + return absl::nullopt; +} + VideoCodec VideoCodec::CreateRtxCodec(int rtx_payload_type, int associated_payload_type) { VideoCodec rtx_codec(rtx_payload_type, kRtxCodecName); diff --git a/media/base/codec.h b/media/base/codec.h index b1a1f07238..f327199484 100644 --- a/media/base/codec.h +++ b/media/base/codec.h @@ -16,6 +16,7 @@ #include #include +#include "absl/types/optional.h" #include "api/rtp_parameters.h" #include "api/video_codecs/sdp_video_format.h" #include "media/base/media_constants.h" @@ -144,6 +145,8 @@ struct AudioCodec : public Codec { }; struct RTC_EXPORT VideoCodec : public Codec { + absl::optional packetization; + // Creates a codec with the given parameters. VideoCodec(int id, const std::string& name); // Creates a codec with the given name and empty id. @@ -171,6 +174,11 @@ struct RTC_EXPORT VideoCodec : public Codec { bool operator!=(const VideoCodec& c) const { return !(*this == c); } + // Return packetization which both |local_codec| and |remote_codec| support. + static absl::optional IntersectPacketization( + const VideoCodec& local_codec, + const VideoCodec& remote_codec); + static VideoCodec CreateRtxCodec(int rtx_payload_type, int associated_payload_type); diff --git a/media/base/codec_unittest.cc b/media/base/codec_unittest.cc index b3cda813b2..bf0addb5a3 100644 --- a/media/base/codec_unittest.cc +++ b/media/base/codec_unittest.cc @@ -174,6 +174,29 @@ TEST(CodecTest, TestVideoCodecOperators) { EXPECT_TRUE(c13 == c10); } +TEST(CodecTest, TestVideoCodecIntersectPacketization) { + VideoCodec c1; + c1.packetization = "raw"; + VideoCodec c2; + c2.packetization = "raw"; + VideoCodec c3; + + EXPECT_EQ(VideoCodec::IntersectPacketization(c1, c2), "raw"); + EXPECT_EQ(VideoCodec::IntersectPacketization(c1, c3), absl::nullopt); +} + +TEST(CodecTest, TestVideoCodecEqualsWithDifferentPacketization) { + VideoCodec c0(100, cricket::kVp8CodecName); + VideoCodec c1(100, cricket::kVp8CodecName); + VideoCodec c2(100, cricket::kVp8CodecName); + c2.packetization = "raw"; + + EXPECT_EQ(c0, c1); + EXPECT_NE(c0, c2); + EXPECT_NE(c2, c0); + EXPECT_EQ(c2, c2); +} + TEST(CodecTest, TestVideoCodecMatches) { // Test a codec with a static payload type. VideoCodec c0(95, "V"); @@ -190,6 +213,15 @@ TEST(CodecTest, TestVideoCodecMatches) { EXPECT_FALSE(c1.Matches(VideoCodec(95, "V"))); } +TEST(CodecTest, TestVideoCodecMatchesWithDifferentPacketization) { + VideoCodec c0(100, cricket::kVp8CodecName); + VideoCodec c1(101, cricket::kVp8CodecName); + c1.packetization = "raw"; + + EXPECT_TRUE(c0.Matches(c1)); + EXPECT_TRUE(c1.Matches(c0)); +} + // VP9 codecs compare profile information. TEST(CodecTest, TestVP9CodecMatches) { const char kProfile0[] = "0"; diff --git a/media/base/media_constants.cc b/media/base/media_constants.cc index 3b4d2f27f8..5bd4b754d2 100644 --- a/media/base/media_constants.cc +++ b/media/base/media_constants.cc @@ -77,6 +77,8 @@ const int kPreferredSPropStereo = 0; const int kPreferredStereo = 0; const int kPreferredUseInbandFec = 0; +const char kPacketizationParamRaw[] = "raw"; + const char kRtcpFbParamLntf[] = "goog-lntf"; const char kRtcpFbParamNack[] = "nack"; const char kRtcpFbNackParamPli[] = "pli"; diff --git a/media/base/media_constants.h b/media/base/media_constants.h index a796474dc3..136e9f19a0 100644 --- a/media/base/media_constants.h +++ b/media/base/media_constants.h @@ -91,6 +91,8 @@ extern const int kPreferredSPropStereo; extern const int kPreferredStereo; extern const int kPreferredUseInbandFec; +extern const char kPacketizationParamRaw[]; + // rtcp-fb message in its first experimental stages. Documentation pending. extern const char kRtcpFbParamLntf[]; // rtcp-fb messages according to RFC 4585 diff --git a/media/engine/webrtc_video_engine.cc b/media/engine/webrtc_video_engine.cc index 4219c438ef..ae366d66cd 100644 --- a/media/engine/webrtc_video_engine.cc +++ b/media/engine/webrtc_video_engine.cc @@ -1877,6 +1877,8 @@ void WebRtcVideoChannel::WebRtcVideoSendStream::SetCodec( parameters_.config.rtp.payload_name = codec_settings.codec.name; parameters_.config.rtp.payload_type = codec_settings.codec.id; + parameters_.config.rtp.raw_payload = + codec_settings.codec.packetization == kPacketizationParamRaw; parameters_.config.rtp.ulpfec = codec_settings.ulpfec; parameters_.config.rtp.flexfec.payload_type = codec_settings.flexfec_payload_type; @@ -2463,6 +2465,7 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::ConfigureCodecs( RTC_DCHECK(!recv_codecs.empty()); config_.decoders.clear(); config_.rtp.rtx_associated_payload_types.clear(); + config_.rtp.raw_payload_types.clear(); for (const auto& recv_codec : recv_codecs) { webrtc::SdpVideoFormat video_format(recv_codec.codec.name, recv_codec.codec.params); @@ -2476,6 +2479,9 @@ void WebRtcVideoChannel::WebRtcVideoReceiveStream::ConfigureCodecs( config_.decoders.push_back(decoder); config_.rtp.rtx_associated_payload_types[recv_codec.rtx_payload_type] = recv_codec.codec.id; + if (recv_codec.codec.packetization == kPacketizationParamRaw) { + config_.rtp.raw_payload_types.insert(recv_codec.codec.id); + } } const auto& codec = recv_codecs.front(); diff --git a/media/engine/webrtc_video_engine_unittest.cc b/media/engine/webrtc_video_engine_unittest.cc index f2b6e373ab..dc94f5dd89 100644 --- a/media/engine/webrtc_video_engine_unittest.cc +++ b/media/engine/webrtc_video_engine_unittest.cc @@ -3617,6 +3617,27 @@ TEST_F(WebRtcVideoChannelTest, SetDefaultSendCodecs) { // TODO(juberti): Check RTCP, PLI, TMMBR. } +TEST_F(WebRtcVideoChannelTest, SetSendCodecsWithoutPacketization) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + + FakeVideoSendStream* stream = AddSendStream(); + const webrtc::VideoSendStream::Config config = stream->GetConfig().Copy(); + EXPECT_FALSE(config.rtp.raw_payload); +} + +TEST_F(WebRtcVideoChannelTest, SetSendCodecsWithPacketization) { + cricket::VideoSendParameters parameters; + parameters.codecs.push_back(GetEngineCodec("VP8")); + parameters.codecs.back().packetization = kPacketizationParamRaw; + EXPECT_TRUE(channel_->SetSendParameters(parameters)); + + FakeVideoSendStream* stream = AddSendStream(); + const webrtc::VideoSendStream::Config config = stream->GetConfig().Copy(); + EXPECT_TRUE(config.rtp.raw_payload); +} + // The following four tests ensures that FlexFEC is not activated by default // when the field trials are not enabled. // TODO(brandtr): Remove or update these tests when FlexFEC _is_ enabled by @@ -4443,6 +4464,42 @@ TEST_F(WebRtcVideoChannelTest, SetRecvCodecsWithRtx) { "rejected."; } +TEST_F(WebRtcVideoChannelTest, SetRecvCodecsWithPacketization) { + cricket::VideoCodec vp8_codec = GetEngineCodec("VP8"); + vp8_codec.packetization = kPacketizationParamRaw; + + cricket::VideoRecvParameters parameters; + parameters.codecs = {vp8_codec, GetEngineCodec("VP9")}; + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + + const cricket::StreamParams params = + cricket::StreamParams::CreateLegacy(kSsrcs1[0]); + AddRecvStream(params); + ASSERT_THAT(fake_call_->GetVideoReceiveStreams(), testing::SizeIs(1)); + + const webrtc::VideoReceiveStream::Config& config = + fake_call_->GetVideoReceiveStreams()[0]->GetConfig(); + ASSERT_THAT(config.rtp.raw_payload_types, testing::SizeIs(1)); + EXPECT_EQ(config.rtp.raw_payload_types.count(vp8_codec.id), 1U); +} + +TEST_F(WebRtcVideoChannelTest, SetRecvCodecsWithPacketizationRecreatesStream) { + cricket::VideoRecvParameters parameters; + parameters.codecs = {GetEngineCodec("VP8"), GetEngineCodec("VP9")}; + parameters.codecs.back().packetization = kPacketizationParamRaw; + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + + const cricket::StreamParams params = + cricket::StreamParams::CreateLegacy(kSsrcs1[0]); + AddRecvStream(params); + ASSERT_THAT(fake_call_->GetVideoReceiveStreams(), testing::SizeIs(1)); + EXPECT_EQ(fake_call_->GetNumCreatedReceiveStreams(), 1); + + parameters.codecs.back().packetization.reset(); + EXPECT_TRUE(channel_->SetRecvParameters(parameters)); + EXPECT_EQ(fake_call_->GetNumCreatedReceiveStreams(), 2); +} + TEST_F(WebRtcVideoChannelTest, SetRecvCodecsWithChangedRtxPayloadType) { const int kUnusedPayloadType1 = 126; const int kUnusedPayloadType2 = 127; diff --git a/pc/channel.cc b/pc/channel.cc index 13341e3e1b..54839631cc 100644 --- a/pc/channel.cc +++ b/pc/channel.cc @@ -1021,6 +1021,26 @@ bool VideoChannel::SetLocalContent_w(const MediaContentDescription* content, VideoRecvParameters recv_params = last_recv_params_; RtpParametersFromMediaDescription(video, rtp_header_extensions, &recv_params); + + VideoSendParameters send_params = last_send_params_; + bool needs_send_params_update = false; + if (type == SdpType::kAnswer || type == SdpType::kPrAnswer) { + for (auto& send_codec : send_params.codecs) { + auto* recv_codec = FindMatchingCodec(recv_params.codecs, send_codec); + if (recv_codec) { + if (!recv_codec->packetization && send_codec.packetization) { + send_codec.packetization.reset(); + needs_send_params_update = true; + } else if (recv_codec->packetization != send_codec.packetization) { + SafeSetError( + "Failed to set local answer due to invalid codec packetization.", + error_desc); + return false; + } + } + } + } + if (!media_channel()->SetRecvParameters(recv_params)) { SafeSetError("Failed to set local video description recv parameters.", error_desc); @@ -1037,6 +1057,14 @@ bool VideoChannel::SetLocalContent_w(const MediaContentDescription* content, last_recv_params_ = recv_params; + if (needs_send_params_update) { + if (!media_channel()->SetSendParameters(send_params)) { + SafeSetError("Failed to set send parameters.", error_desc); + return false; + } + last_send_params_ = send_params; + } + // TODO(pthatcher): Move local streams into VideoSendParameters, and // only give it to the media channel once we have a remote // description too (without a remote description, we won't be able @@ -1077,15 +1105,40 @@ bool VideoChannel::SetRemoteContent_w(const MediaContentDescription* content, } send_params.mid = content_name(); - bool parameters_applied = media_channel()->SetSendParameters(send_params); + VideoRecvParameters recv_params = last_recv_params_; + bool needs_recv_params_update = false; + if (type == SdpType::kAnswer || type == SdpType::kPrAnswer) { + for (auto& recv_codec : recv_params.codecs) { + auto* send_codec = FindMatchingCodec(send_params.codecs, recv_codec); + if (send_codec) { + if (!send_codec->packetization && recv_codec.packetization) { + recv_codec.packetization.reset(); + needs_recv_params_update = true; + } else if (send_codec->packetization != recv_codec.packetization) { + SafeSetError( + "Failed to set remote answer due to invalid codec packetization.", + error_desc); + return false; + } + } + } + } - if (!parameters_applied) { + if (!media_channel()->SetSendParameters(send_params)) { SafeSetError("Failed to set remote video description send parameters.", error_desc); return false; } last_send_params_ = send_params; + if (needs_recv_params_update) { + if (!media_channel()->SetRecvParameters(recv_params)) { + SafeSetError("Failed to set recv parameters.", error_desc); + return false; + } + last_recv_params_ = recv_params; + } + // TODO(pthatcher): Move remote streams into VideoRecvParameters, // and only give it to the media channel once we have a local // description too (without a local description, we won't be able to diff --git a/pc/channel_unittest.cc b/pc/channel_unittest.cc index e4252b2e8b..db0e8a8d02 100644 --- a/pc/channel_unittest.cc +++ b/pc/channel_unittest.cc @@ -37,6 +37,7 @@ #include "rtc_base/checks.h" #include "rtc_base/rtc_certificate.h" #include "rtc_base/ssl_identity.h" +#include "test/gmock.h" #include "test/gtest.h" using cricket::DtlsTransportInternal; @@ -2121,6 +2122,152 @@ TEST_F(VideoChannelSingleThreadTest, UpdateLocalStreamsWithSimulcast) { Base::TestUpdateLocalStreamsWithSimulcast(); } +TEST_F(VideoChannelSingleThreadTest, TestSetLocalOfferWithPacketization) { + const cricket::VideoCodec kVp8Codec(97, "VP8"); + cricket::VideoCodec vp9_codec(98, "VP9"); + vp9_codec.packetization = cricket::kPacketizationParamRaw; + cricket::VideoContentDescription video; + video.set_codecs({kVp8Codec, vp9_codec}); + + CreateChannels(0, 0); + + EXPECT_TRUE(channel1_->SetLocalContent(&video, SdpType::kOffer, NULL)); + EXPECT_THAT(media_channel1_->send_codecs(), testing::IsEmpty()); + ASSERT_THAT(media_channel1_->recv_codecs(), testing::SizeIs(2)); + EXPECT_TRUE(media_channel1_->recv_codecs()[0].Matches(kVp8Codec)); + EXPECT_EQ(media_channel1_->recv_codecs()[0].packetization, absl::nullopt); + EXPECT_TRUE(media_channel1_->recv_codecs()[1].Matches(vp9_codec)); + EXPECT_EQ(media_channel1_->recv_codecs()[1].packetization, + cricket::kPacketizationParamRaw); +} + +TEST_F(VideoChannelSingleThreadTest, TestSetRemoteOfferWithPacketization) { + const cricket::VideoCodec kVp8Codec(97, "VP8"); + cricket::VideoCodec vp9_codec(98, "VP9"); + vp9_codec.packetization = cricket::kPacketizationParamRaw; + cricket::VideoContentDescription video; + video.set_codecs({kVp8Codec, vp9_codec}); + + CreateChannels(0, 0); + + EXPECT_TRUE(channel1_->SetRemoteContent(&video, SdpType::kOffer, NULL)); + EXPECT_THAT(media_channel1_->recv_codecs(), testing::IsEmpty()); + ASSERT_THAT(media_channel1_->send_codecs(), testing::SizeIs(2)); + EXPECT_TRUE(media_channel1_->send_codecs()[0].Matches(kVp8Codec)); + EXPECT_EQ(media_channel1_->send_codecs()[0].packetization, absl::nullopt); + EXPECT_TRUE(media_channel1_->send_codecs()[1].Matches(vp9_codec)); + EXPECT_EQ(media_channel1_->send_codecs()[1].packetization, + cricket::kPacketizationParamRaw); +} + +TEST_F(VideoChannelSingleThreadTest, TestSetAnswerWithPacketization) { + const cricket::VideoCodec kVp8Codec(97, "VP8"); + cricket::VideoCodec vp9_codec(98, "VP9"); + vp9_codec.packetization = cricket::kPacketizationParamRaw; + cricket::VideoContentDescription video; + video.set_codecs({kVp8Codec, vp9_codec}); + + CreateChannels(0, 0); + + EXPECT_TRUE(channel1_->SetLocalContent(&video, SdpType::kOffer, NULL)); + EXPECT_TRUE(channel1_->SetRemoteContent(&video, SdpType::kAnswer, NULL)); + ASSERT_THAT(media_channel1_->recv_codecs(), testing::SizeIs(2)); + EXPECT_TRUE(media_channel1_->recv_codecs()[0].Matches(kVp8Codec)); + EXPECT_EQ(media_channel1_->recv_codecs()[0].packetization, absl::nullopt); + EXPECT_TRUE(media_channel1_->recv_codecs()[1].Matches(vp9_codec)); + EXPECT_EQ(media_channel1_->recv_codecs()[1].packetization, + cricket::kPacketizationParamRaw); + EXPECT_THAT(media_channel1_->send_codecs(), testing::SizeIs(2)); + EXPECT_TRUE(media_channel1_->send_codecs()[0].Matches(kVp8Codec)); + EXPECT_EQ(media_channel1_->send_codecs()[0].packetization, absl::nullopt); + EXPECT_TRUE(media_channel1_->send_codecs()[1].Matches(vp9_codec)); + EXPECT_EQ(media_channel1_->send_codecs()[1].packetization, + cricket::kPacketizationParamRaw); +} + +TEST_F(VideoChannelSingleThreadTest, TestSetLocalAnswerWithoutPacketization) { + const cricket::VideoCodec kLocalCodec(98, "VP8"); + cricket::VideoCodec remote_codec(99, "VP8"); + remote_codec.packetization = cricket::kPacketizationParamRaw; + cricket::VideoContentDescription local_video; + local_video.set_codecs({kLocalCodec}); + cricket::VideoContentDescription remote_video; + remote_video.set_codecs({remote_codec}); + + CreateChannels(0, 0); + + EXPECT_TRUE( + channel1_->SetRemoteContent(&remote_video, SdpType::kOffer, NULL)); + EXPECT_TRUE(channel1_->SetLocalContent(&local_video, SdpType::kAnswer, NULL)); + ASSERT_THAT(media_channel1_->recv_codecs(), testing::SizeIs(1)); + EXPECT_EQ(media_channel1_->recv_codecs()[0].packetization, absl::nullopt); + ASSERT_THAT(media_channel1_->send_codecs(), testing::SizeIs(1)); + EXPECT_EQ(media_channel1_->send_codecs()[0].packetization, absl::nullopt); +} + +TEST_F(VideoChannelSingleThreadTest, TestSetRemoteAnswerWithoutPacketization) { + cricket::VideoCodec local_codec(98, "VP8"); + local_codec.packetization = cricket::kPacketizationParamRaw; + const cricket::VideoCodec kRemoteCodec(99, "VP8"); + cricket::VideoContentDescription local_video; + local_video.set_codecs({local_codec}); + cricket::VideoContentDescription remote_video; + remote_video.set_codecs({kRemoteCodec}); + + CreateChannels(0, 0); + + EXPECT_TRUE(channel1_->SetLocalContent(&local_video, SdpType::kOffer, NULL)); + EXPECT_TRUE( + channel1_->SetRemoteContent(&remote_video, SdpType::kAnswer, NULL)); + ASSERT_THAT(media_channel1_->recv_codecs(), testing::SizeIs(1)); + EXPECT_EQ(media_channel1_->recv_codecs()[0].packetization, absl::nullopt); + ASSERT_THAT(media_channel1_->send_codecs(), testing::SizeIs(1)); + EXPECT_EQ(media_channel1_->send_codecs()[0].packetization, absl::nullopt); +} + +TEST_F(VideoChannelSingleThreadTest, + TestSetRemoteAnswerWithInvalidPacketization) { + cricket::VideoCodec local_codec(98, "VP8"); + local_codec.packetization = cricket::kPacketizationParamRaw; + cricket::VideoCodec remote_codec(99, "VP8"); + remote_codec.packetization = "unknownpacketizationattributevalue"; + cricket::VideoContentDescription local_video; + local_video.set_codecs({local_codec}); + cricket::VideoContentDescription remote_video; + remote_video.set_codecs({remote_codec}); + + CreateChannels(0, 0); + + EXPECT_TRUE(channel1_->SetLocalContent(&local_video, SdpType::kOffer, NULL)); + EXPECT_FALSE( + channel1_->SetRemoteContent(&remote_video, SdpType::kAnswer, NULL)); + ASSERT_THAT(media_channel1_->recv_codecs(), testing::SizeIs(1)); + EXPECT_EQ(media_channel1_->recv_codecs()[0].packetization, + cricket::kPacketizationParamRaw); + EXPECT_THAT(media_channel1_->send_codecs(), testing::IsEmpty()); +} + +TEST_F(VideoChannelSingleThreadTest, + TestSetLocalAnswerWithInvalidPacketization) { + cricket::VideoCodec local_codec(98, "VP8"); + local_codec.packetization = cricket::kPacketizationParamRaw; + const cricket::VideoCodec kRemoteCodec(99, "VP8"); + cricket::VideoContentDescription local_video; + local_video.set_codecs({local_codec}); + cricket::VideoContentDescription remote_video; + remote_video.set_codecs({kRemoteCodec}); + + CreateChannels(0, 0); + + EXPECT_TRUE( + channel1_->SetRemoteContent(&remote_video, SdpType::kOffer, NULL)); + EXPECT_FALSE( + channel1_->SetLocalContent(&local_video, SdpType::kAnswer, NULL)); + EXPECT_THAT(media_channel1_->recv_codecs(), testing::IsEmpty()); + ASSERT_THAT(media_channel1_->send_codecs(), testing::SizeIs(1)); + EXPECT_EQ(media_channel1_->send_codecs()[0].packetization, absl::nullopt); +} + // VideoChannelDoubleThreadTest TEST_F(VideoChannelDoubleThreadTest, TestInit) { Base::TestInit(); diff --git a/pc/media_session.cc b/pc/media_session.cc index 7cc4b95f14..fe11d44702 100644 --- a/pc/media_session.cc +++ b/pc/media_session.cc @@ -786,6 +786,19 @@ static bool ReferencedCodecsMatch(const std::vector& codecs1, return codec1 != nullptr && codec2 != nullptr && codec1->Matches(*codec2); } +template +static void NegotiatePacketization(const C& local_codec, + const C& remote_codec, + C* negotiated_codec) {} + +template <> +void NegotiatePacketization(const VideoCodec& local_codec, + const VideoCodec& remote_codec, + VideoCodec* negotiated_codec) { + negotiated_codec->packetization = + VideoCodec::IntersectPacketization(local_codec, remote_codec); +} + template static void NegotiateCodecs(const std::vector& local_codecs, const std::vector& offered_codecs, @@ -797,6 +810,7 @@ static void NegotiateCodecs(const std::vector& local_codecs, // local codecs, in case the remote offer contains duplicate codecs. if (FindMatchingCodec(local_codecs, offered_codecs, ours, &theirs)) { C negotiated = ours; + NegotiatePacketization(ours, theirs, &negotiated); negotiated.IntersectFeedbackParams(theirs); if (IsRtxCodec(negotiated)) { const auto apt_it = @@ -2187,6 +2201,14 @@ bool MediaSessionDescriptionFactory::AddVideoContentForOffer( } } + if (session_options.raw_packetization_for_video) { + for (VideoCodec& codec : filtered_codecs) { + if (codec.GetCodecType() == VideoCodec::CODEC_VIDEO) { + codec.packetization = kPacketizationParamRaw; + } + } + } + if (!CreateMediaContentOffer(media_description_options, session_options, filtered_codecs, sdes_policy, GetCryptos(current_content), crypto_suites, @@ -2504,6 +2526,14 @@ bool MediaSessionDescriptionFactory::AddVideoContentForAnswer( } } + if (session_options.raw_packetization_for_video) { + for (VideoCodec& codec : filtered_codecs) { + if (codec.GetCodecType() == VideoCodec::CODEC_VIDEO) { + codec.packetization = kPacketizationParamRaw; + } + } + } + bool bundle_enabled = offer_description->HasGroup(GROUP_TYPE_BUNDLE) && session_options.bundle_enabled; diff --git a/pc/media_session.h b/pc/media_session.h index 8ba6293b66..b154a1816d 100644 --- a/pc/media_session.h +++ b/pc/media_session.h @@ -106,6 +106,7 @@ struct MediaSessionOptions { bool rtcp_mux_enabled = true; bool bundle_enabled = false; bool offer_extmap_allow_mixed = false; + bool raw_packetization_for_video = false; std::string rtcp_cname = kDefaultRtcpCname; webrtc::CryptoOptions crypto_options; // List of media description options in the same order that the media diff --git a/pc/peer_connection.cc b/pc/peer_connection.cc index 82eb425d4d..fde0433278 100644 --- a/pc/peer_connection.cc +++ b/pc/peer_connection.cc @@ -858,6 +858,8 @@ void ExtractSharedMediaSessionOptions( cricket::MediaSessionOptions* session_options) { session_options->vad_enabled = rtc_options.voice_activity_detection; session_options->bundle_enabled = rtc_options.use_rtp_mux; + session_options->raw_packetization_for_video = + rtc_options.raw_packetization_for_video; } PeerConnection::PeerConnection(PeerConnectionFactory* factory, diff --git a/pc/peer_connection_media_unittest.cc b/pc/peer_connection_media_unittest.cc index 1ff9f07198..ac2563d5fd 100644 --- a/pc/peer_connection_media_unittest.cc +++ b/pc/peer_connection_media_unittest.cc @@ -15,6 +15,7 @@ #include #include "absl/algorithm/container.h" +#include "absl/types/optional.h" #include "api/call/call_factory_interface.h" #include "api/test/fake_media_transport.h" #include "logging/rtc_event_log/rtc_event_log_factory.h" @@ -516,6 +517,106 @@ TEST_P(PeerConnectionMediaTest, EXPECT_FALSE(video_content->rejected); } +// Test that raw packetization is not set in the offer by default. +TEST_P(PeerConnectionMediaTest, RawPacketizationNotSetInOffer) { + std::vector fake_codecs; + fake_codecs.push_back(cricket::VideoCodec(111, cricket::kVp8CodecName)); + fake_codecs.push_back(cricket::VideoCodec(112, cricket::kRtxCodecName)); + fake_codecs.back().params[cricket::kCodecParamAssociatedPayloadType] = "111"; + fake_codecs.push_back(cricket::VideoCodec(113, cricket::kVp9CodecName)); + fake_codecs.push_back(cricket::VideoCodec(114, cricket::kH264CodecName)); + fake_codecs.push_back(cricket::VideoCodec(115, "HEVC")); + auto caller_fake_engine = absl::make_unique(); + caller_fake_engine->SetVideoCodecs(fake_codecs); + + auto caller = CreatePeerConnectionWithVideo(std::move(caller_fake_engine)); + auto offer = caller->CreateOfferAndSetAsLocal(); + auto* offer_description = + cricket::GetFirstVideoContentDescription(offer->description()); + for (const auto& codec : offer_description->codecs()) { + EXPECT_EQ(codec.packetization, absl::nullopt); + } +} + +// Test that raw packetization is set in the offer and answer for all +// video payload when raw_packetization_for_video is true. +TEST_P(PeerConnectionMediaTest, RawPacketizationSetInOfferAndAnswer) { + std::vector fake_codecs; + fake_codecs.push_back(cricket::VideoCodec(111, cricket::kVp8CodecName)); + fake_codecs.push_back(cricket::VideoCodec(112, cricket::kRtxCodecName)); + fake_codecs.back().params[cricket::kCodecParamAssociatedPayloadType] = "111"; + fake_codecs.push_back(cricket::VideoCodec(113, cricket::kVp9CodecName)); + fake_codecs.push_back(cricket::VideoCodec(114, cricket::kH264CodecName)); + fake_codecs.push_back(cricket::VideoCodec(115, "HEVC")); + auto caller_fake_engine = absl::make_unique(); + caller_fake_engine->SetVideoCodecs(fake_codecs); + auto callee_fake_engine = absl::make_unique(); + callee_fake_engine->SetVideoCodecs(fake_codecs); + + RTCOfferAnswerOptions options; + options.raw_packetization_for_video = true; + + auto caller = CreatePeerConnectionWithVideo(std::move(caller_fake_engine)); + auto offer = caller->CreateOfferAndSetAsLocal(options); + auto* offer_description = + cricket::GetFirstVideoContentDescription(offer->description()); + for (const auto& codec : offer_description->codecs()) { + if (codec.GetCodecType() == cricket::VideoCodec::CODEC_VIDEO) { + EXPECT_EQ(codec.packetization, cricket::kPacketizationParamRaw); + } + } + + auto callee = CreatePeerConnectionWithVideo(std::move(callee_fake_engine)); + ASSERT_TRUE(callee->SetRemoteDescription(std::move(offer))); + auto answer = callee->CreateAnswerAndSetAsLocal(options); + auto* answer_description = + cricket::GetFirstVideoContentDescription(answer->description()); + for (const auto& codec : answer_description->codecs()) { + if (codec.GetCodecType() == cricket::VideoCodec::CODEC_VIDEO) { + EXPECT_EQ(codec.packetization, cricket::kPacketizationParamRaw); + } + } + + ASSERT_TRUE(caller->SetRemoteDescription(std::move(answer))); +} + +// Test that raw packetization is not set in the answer when +// raw_packetization_for_video is true if it was not set in the offer. +TEST_P(PeerConnectionMediaTest, + RawPacketizationNotSetInAnswerWhenNotSetInOffer) { + std::vector fake_codecs; + fake_codecs.push_back(cricket::VideoCodec(111, cricket::kVp8CodecName)); + fake_codecs.push_back(cricket::VideoCodec(112, cricket::kRtxCodecName)); + fake_codecs.back().params[cricket::kCodecParamAssociatedPayloadType] = "111"; + fake_codecs.push_back(cricket::VideoCodec(113, cricket::kVp9CodecName)); + fake_codecs.push_back(cricket::VideoCodec(114, cricket::kH264CodecName)); + fake_codecs.push_back(cricket::VideoCodec(115, "HEVC")); + auto caller_fake_engine = absl::make_unique(); + caller_fake_engine->SetVideoCodecs(fake_codecs); + auto callee_fake_engine = absl::make_unique(); + callee_fake_engine->SetVideoCodecs(fake_codecs); + + RTCOfferAnswerOptions caller_options; + caller_options.raw_packetization_for_video = false; + RTCOfferAnswerOptions callee_options; + callee_options.raw_packetization_for_video = true; + + auto caller = CreatePeerConnectionWithVideo(std::move(caller_fake_engine)); + auto offer = caller->CreateOfferAndSetAsLocal(caller_options); + + auto callee = CreatePeerConnectionWithVideo(std::move(callee_fake_engine)); + ASSERT_TRUE(callee->SetRemoteDescription(std::move(offer))); + auto answer = callee->CreateAnswerAndSetAsLocal(callee_options); + + auto* answer_description = + cricket::GetFirstVideoContentDescription(answer->description()); + for (const auto& codec : answer_description->codecs()) { + EXPECT_EQ(codec.packetization, absl::nullopt); + } + + ASSERT_TRUE(caller->SetRemoteDescription(std::move(answer))); +} + class PeerConnectionMediaOfferDirectionTest : public PeerConnectionMediaBaseTest, public ::testing::WithParamInterface< diff --git a/pc/webrtc_sdp.cc b/pc/webrtc_sdp.cc index af8a5277de..83bd3782bd 100644 --- a/pc/webrtc_sdp.cc +++ b/pc/webrtc_sdp.cc @@ -180,6 +180,7 @@ static const char kAttributeSimulcast[] = "simulcast"; // draft-ietf-mmusic-rid-15 // a=rid static const char kAttributeRid[] = "rid"; +static const char kAttributePacketization[] = "packetization"; // Experimental flags static const char kAttributeXGoogleFlag[] = "x-google-flag"; @@ -344,6 +345,10 @@ static bool ParseFmtpParam(const std::string& line, std::string* parameter, std::string* value, SdpParseError* error); +static bool ParsePacketizationAttribute(const std::string& line, + const cricket::MediaType media_type, + MediaContentDescription* media_desc, + SdpParseError* error); static bool ParseRtcpFbAttribute(const std::string& line, const cricket::MediaType media_type, MediaContentDescription* media_desc, @@ -1744,6 +1749,14 @@ void WriteFmtpHeader(int payload_type, rtc::StringBuilder* os) { *os << kSdpDelimiterColon << payload_type; } +void WritePacketizationHeader(int payload_type, rtc::StringBuilder* os) { + // packetization header: a=packetization:|payload_type| + // Add a=packetization + InitAttrLine(kAttributePacketization, os); + // Add :|payload_type| + *os << kSdpDelimiterColon << payload_type; +} + void WriteRtcpFbHeader(int payload_type, rtc::StringBuilder* os) { // rtcp-fb header: a=rtcp-fb:|payload_type| // /> @@ -1819,6 +1832,17 @@ void AddFmtpLine(const T& codec, std::string* message) { return; } +template +void AddPacketizationLine(const T& codec, std::string* message) { + if (!codec.packetization) { + return; + } + rtc::StringBuilder os; + WritePacketizationHeader(codec.id, &os); + os << " " << *codec.packetization; + AddLine(os.str(), message); +} + template void AddRtcpFbLines(const T& codec, std::string* message) { for (const cricket::FeedbackParam& param : codec.feedback_params.params()) { @@ -1871,6 +1895,7 @@ void BuildRtpMap(const MediaContentDescription* media_desc, << cricket::kVideoCodecClockrate; AddLine(os.str(), message); } + AddPacketizationLine(codec, message); AddRtcpFbLines(codec, message); AddFmtpLine(codec, message); } @@ -2910,6 +2935,24 @@ void UpdateCodec(MediaContentDescription* content_desc, AddOrReplaceCodec(content_desc, new_codec); } +// Adds or updates existing video codec corresponding to |payload_type| +// according to |packetization|. +void UpdateVideoCodecPacketization(VideoContentDescription* video_desc, + int payload_type, + const std::string& packetization) { + if (packetization != cricket::kPacketizationParamRaw) { + // Ignore unsupported packetization attribute. + return; + } + + // Codec might already have been populated (from rtpmap). + cricket::VideoCodec codec = + GetCodecWithPayloadType(video_desc->codecs(), payload_type); + codec.packetization = packetization; + AddOrReplaceCodec(video_desc, + codec); +} + template bool PopWildcardCodec(std::vector* codecs, T* wildcard_codec) { for (auto iter = codecs->begin(); iter != codecs->end(); ++iter) { @@ -3166,6 +3209,10 @@ bool ParseContent(const std::string& message, if (!GetValue(line, kCodecParamMaxPTime, &maxptime_as_string, error)) { return false; } + } else if (HasAttribute(line, kAttributePacketization)) { + if (!ParsePacketizationAttribute(line, media_type, media_desc, error)) { + return false; + } } else if (HasAttribute(line, kAttributeRtcpFb)) { if (!ParseRtcpFbAttribute(line, media_type, media_desc, error)) { return false; @@ -3682,6 +3729,34 @@ bool ParseFmtpAttributes(const std::string& line, return true; } +bool ParsePacketizationAttribute(const std::string& line, + const cricket::MediaType media_type, + MediaContentDescription* media_desc, + SdpParseError* error) { + if (media_type != cricket::MEDIA_TYPE_VIDEO) { + return true; + } + std::vector packetization_fields; + rtc::split(line.c_str(), kSdpDelimiterSpaceChar, &packetization_fields); + if (packetization_fields.size() < 2) { + return ParseFailedGetValue(line, kAttributePacketization, error); + } + std::string payload_type_string; + if (!GetValue(packetization_fields[0], kAttributePacketization, + &payload_type_string, error)) { + return false; + } + int payload_type; + if (!GetPayloadTypeFromString(line, payload_type_string, &payload_type, + error)) { + return false; + } + std::string packetization = packetization_fields[1]; + UpdateVideoCodecPacketization(media_desc->as_video(), payload_type, + packetization); + return true; +} + bool ParseRtcpFbAttribute(const std::string& line, const cricket::MediaType media_type, MediaContentDescription* media_desc, diff --git a/pc/webrtc_sdp_unittest.cc b/pc/webrtc_sdp_unittest.cc index 4b7a6d9219..5fdf5f61ab 100644 --- a/pc/webrtc_sdp_unittest.cc +++ b/pc/webrtc_sdp_unittest.cc @@ -3418,6 +3418,54 @@ TEST_F(WebRtcSdpTest, DeserializeVideoFmtpWithSpace) { EXPECT_EQ(found->second, "40"); } +TEST_F(WebRtcSdpTest, DeserializePacketizationAttributeWithIllegalValue) { + JsepSessionDescription jdesc_output(kDummyType); + + const char kSdpWithPacketizationString[] = + "v=0\r\n" + "o=- 18446744069414584320 18446462598732840960 IN IP4 127.0.0.1\r\n" + "s=-\r\n" + "t=0 0\r\n" + "m=audio 9 RTP/SAVPF 111\r\n" + "a=rtpmap:111 opus/48000/2\r\n" + "a=packetization:111 unknownpacketizationattributeforaudio\r\n" + "m=video 3457 RTP/SAVPF 120 121 122\r\n" + "a=rtpmap:120 VP8/90000\r\n" + "a=packetization:120 raw\r\n" + "a=rtpmap:121 VP9/90000\r\n" + "a=rtpmap:122 H264/90000\r\n" + "a=packetization:122 unknownpacketizationattributevalue\r\n"; + + SdpParseError error; + EXPECT_TRUE(webrtc::SdpDeserialize(kSdpWithPacketizationString, &jdesc_output, + &error)); + + AudioContentDescription* acd = + GetFirstAudioContentDescription(jdesc_output.description()); + ASSERT_TRUE(acd); + ASSERT_THAT(acd->codecs(), testing::SizeIs(1)); + cricket::AudioCodec opus = acd->codecs()[0]; + EXPECT_EQ(opus.name, "opus"); + EXPECT_EQ(opus.id, 111); + + const VideoContentDescription* vcd = + GetFirstVideoContentDescription(jdesc_output.description()); + ASSERT_TRUE(vcd); + ASSERT_THAT(vcd->codecs(), testing::SizeIs(3)); + cricket::VideoCodec vp8 = vcd->codecs()[0]; + EXPECT_EQ(vp8.name, "VP8"); + EXPECT_EQ(vp8.id, 120); + EXPECT_EQ(vp8.packetization, "raw"); + cricket::VideoCodec vp9 = vcd->codecs()[1]; + EXPECT_EQ(vp9.name, "VP9"); + EXPECT_EQ(vp9.id, 121); + EXPECT_EQ(vp9.packetization, absl::nullopt); + cricket::VideoCodec h264 = vcd->codecs()[2]; + EXPECT_EQ(h264.name, "H264"); + EXPECT_EQ(h264.id, 122); + EXPECT_EQ(h264.packetization, absl::nullopt); +} + TEST_F(WebRtcSdpTest, SerializeAudioFmtpWithUnknownParameter) { AudioContentDescription* acd = GetFirstAudioContentDescription(&desc_); @@ -3486,6 +3534,22 @@ TEST_F(WebRtcSdpTest, SerializeVideoFmtp) { EXPECT_EQ(sdp_with_fmtp, message); } +TEST_F(WebRtcSdpTest, SerializeVideoPacketizationAttribute) { + VideoContentDescription* vcd = GetFirstVideoContentDescription(&desc_); + + cricket::VideoCodecs codecs = vcd->codecs(); + codecs[0].packetization = "raw"; + vcd->set_codecs(codecs); + + ASSERT_TRUE(jdesc_.Initialize(desc_.Clone(), jdesc_.session_id(), + jdesc_.session_version())); + std::string message = webrtc::SdpSerialize(jdesc_); + std::string sdp_with_packetization = kSdpFullString; + InjectAfter("a=rtpmap:120 VP8/90000\r\n", "a=packetization:120 raw\r\n", + &sdp_with_packetization); + EXPECT_EQ(sdp_with_packetization, message); +} + TEST_F(WebRtcSdpTest, DeserializeAndSerializeSdpWithIceLite) { // Deserialize the baseline description, making sure it's ICE full. JsepSessionDescription jdesc_with_icelite(kDummyType); diff --git a/sdk/media_constraints.cc b/sdk/media_constraints.cc index 7c5fe4a258..56d9e70ee2 100644 --- a/sdk/media_constraints.cc +++ b/sdk/media_constraints.cc @@ -134,6 +134,9 @@ const char MediaConstraints::kScreencastMinBitrate[] = // TODO(ronghuawu): Remove once cpu overuse detection is stable. const char MediaConstraints::kCpuOveruseDetection[] = "googCpuOveruseDetection"; +const char MediaConstraints::kRawPacketizationForVideoEnabled[] = + "googRawPacketizationForVideoEnabled"; + const char MediaConstraints::kNumSimulcastLayers[] = "googNumSimulcastLayers"; // Set |value| to the value associated with the first appearance of |key|, or @@ -262,6 +265,12 @@ bool CopyConstraintsIntoOfferAnswerOptions( offer_answer_options->ice_restart = value; } + if (FindConstraint(constraints, + MediaConstraints::kRawPacketizationForVideoEnabled, &value, + &mandatory_constraints_satisfied)) { + offer_answer_options->raw_packetization_for_video = value; + } + int layers; if (FindConstraint(constraints, MediaConstraints::kNumSimulcastLayers, &layers, &mandatory_constraints_satisfied)) { diff --git a/sdk/media_constraints.h b/sdk/media_constraints.h index 3a151417eb..f3e9853971 100644 --- a/sdk/media_constraints.h +++ b/sdk/media_constraints.h @@ -102,6 +102,10 @@ class MediaConstraints { static const char kScreencastMinBitrate[]; // googScreencastMinBitrate static const char kCpuOveruseDetection[]; // googCpuOveruseDetection + // Constraint to enable negotiating raw RTP packetization using attribute + // "a=packetization: raw" in the SDP for all video payload. + static const char kRawPacketizationForVideoEnabled[]; + // Specifies number of simulcast layers for all video tracks // with a Plan B offer/answer // (see RTCOfferAnswerOptions::num_simulcast_layers).