diff --git a/api/BUILD.gn b/api/BUILD.gn index 72a5eb84ff..9b7301002d 100644 --- a/api/BUILD.gn +++ b/api/BUILD.gn @@ -453,8 +453,10 @@ rtc_library("rtp_parameters") { "../rtc_base:checks", "../rtc_base:stringutils", "../rtc_base/system:rtc_export", + "video_codecs:scalability_mode", ] absl_deps = [ + "//third_party/abseil-cpp/absl/container:inlined_vector", "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] diff --git a/api/rtp_parameters.h b/api/rtp_parameters.h index 45cedfdd9c..e311577878 100644 --- a/api/rtp_parameters.h +++ b/api/rtp_parameters.h @@ -17,11 +17,13 @@ #include #include +#include "absl/container/inlined_vector.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "api/media_types.h" #include "api/priority.h" #include "api/rtp_transceiver_direction.h" +#include "api/video_codecs/scalability_mode.h" #include "rtc_base/system/rtc_export.h" namespace webrtc { @@ -186,6 +188,9 @@ struct RTC_EXPORT RtpCodecCapability { // TODO(deadbeef): Not implemented. bool svc_multi_stream_support = false; + // https://w3c.github.io/webrtc-svc/#dom-rtcrtpcodeccapability-scalabilitymodes + absl::InlinedVector scalability_modes; + bool operator==(const RtpCodecCapability& o) const { return name == o.name && kind == o.kind && clock_rate == o.clock_rate && preferred_payload_type == o.preferred_payload_type && @@ -194,7 +199,8 @@ struct RTC_EXPORT RtpCodecCapability { parameters == o.parameters && options == o.options && max_temporal_layer_extensions == o.max_temporal_layer_extensions && max_spatial_layer_extensions == o.max_spatial_layer_extensions && - svc_multi_stream_support == o.svc_multi_stream_support; + svc_multi_stream_support == o.svc_multi_stream_support && + scalability_modes == o.scalability_modes; } bool operator!=(const RtpCodecCapability& o) const { return !(*this == o); } }; diff --git a/api/video_codecs/BUILD.gn b/api/video_codecs/BUILD.gn index 1671af172c..0b6db8711d 100644 --- a/api/video_codecs/BUILD.gn +++ b/api/video_codecs/BUILD.gn @@ -14,7 +14,15 @@ if (is_android) { rtc_source_set("scalability_mode") { visibility = [ "*" ] - sources = [ "scalability_mode.h" ] + sources = [ + "scalability_mode.cc", + "scalability_mode.h", + ] + deps = [ + "../../rtc_base:checks", + "../../rtc_base/system:rtc_export", + ] + absl_deps = [ "//third_party/abseil-cpp/absl/strings" ] } rtc_library("video_codecs_api") { @@ -144,6 +152,8 @@ rtc_source_set("video_encoder_factory_template_libvpx_vp8_adapter") { "../../modules/video_coding:webrtc_vp8", "../../modules/video_coding:webrtc_vp8_scalability", ] + + absl_deps = [ "//third_party/abseil-cpp/absl/container:inlined_vector" ] } rtc_source_set("video_encoder_factory_template_libvpx_vp9_adapter") { @@ -173,6 +183,7 @@ rtc_source_set("video_encoder_factory_template_libaom_av1_adapter") { "../../modules/video_coding/codecs/av1:libaom_av1_encoder", "../../modules/video_coding/svc:scalability_mode_util", ] + absl_deps = [ "//third_party/abseil-cpp/absl/container:inlined_vector" ] } rtc_library("vp8_temporal_layers_factory") { diff --git a/api/video_codecs/scalability_mode.cc b/api/video_codecs/scalability_mode.cc new file mode 100644 index 0000000000..7dc8fe3e9c --- /dev/null +++ b/api/video_codecs/scalability_mode.cc @@ -0,0 +1,77 @@ +/* + * Copyright (c) 2022 The WebRTC project authors. All Rights Reserved. + * + * Use of this source code is governed by a BSD-style license + * that can be found in the LICENSE file in the root of the source + * tree. An additional intellectual property rights grant can be found + * in the file PATENTS. All contributing project authors may + * be found in the AUTHORS file in the root of the source tree. + */ + +#include "api/video_codecs/scalability_mode.h" + +#include "rtc_base/checks.h" + +namespace webrtc { + +absl::string_view ScalabilityModeToString(ScalabilityMode scalability_mode) { + switch (scalability_mode) { + case ScalabilityMode::kL1T1: + return "L1T1"; + case ScalabilityMode::kL1T2: + return "L1T2"; + case ScalabilityMode::kL1T2h: + return "L1T2h"; + case ScalabilityMode::kL1T3: + return "L1T3"; + case ScalabilityMode::kL1T3h: + return "L1T3h"; + case ScalabilityMode::kL2T1: + return "L2T1"; + case ScalabilityMode::kL2T1h: + return "L2T1h"; + case ScalabilityMode::kL2T1_KEY: + return "L2T1_KEY"; + case ScalabilityMode::kL2T2: + return "L2T2"; + case ScalabilityMode::kL2T2h: + return "L2T2h"; + case ScalabilityMode::kL2T2_KEY: + return "L2T2_KEY"; + case ScalabilityMode::kL2T2_KEY_SHIFT: + return "L2T2_KEY_SHIFT"; + case ScalabilityMode::kL2T3: + return "L2T3"; + case ScalabilityMode::kL2T3h: + return "L2T3h"; + case ScalabilityMode::kL2T3_KEY: + return "L2T3_KEY"; + case ScalabilityMode::kL3T1: + return "L3T1"; + case ScalabilityMode::kL3T1h: + return "L3T1h"; + case ScalabilityMode::kL3T1_KEY: + return "L3T1_KEY"; + case ScalabilityMode::kL3T2: + return "L3T2"; + case ScalabilityMode::kL3T2h: + return "L3T2h"; + case ScalabilityMode::kL3T2_KEY: + return "L3T2_KEY"; + case ScalabilityMode::kL3T3: + return "L3T3"; + case ScalabilityMode::kL3T3h: + return "L3T3h"; + case ScalabilityMode::kL3T3_KEY: + return "L3T3_KEY"; + case ScalabilityMode::kS2T1: + return "S2T1"; + case ScalabilityMode::kS2T3: + return "S2T3"; + case ScalabilityMode::kS3T3: + return "S3T3"; + } + RTC_CHECK_NOTREACHED(); +} + +} // namespace webrtc diff --git a/api/video_codecs/scalability_mode.h b/api/video_codecs/scalability_mode.h index a33b9e420d..397e3d3793 100644 --- a/api/video_codecs/scalability_mode.h +++ b/api/video_codecs/scalability_mode.h @@ -11,6 +11,12 @@ #ifndef API_VIDEO_CODECS_SCALABILITY_MODE_H_ #define API_VIDEO_CODECS_SCALABILITY_MODE_H_ +#include +#include + +#include "absl/strings/string_view.h" +#include "rtc_base/system/rtc_export.h" + namespace webrtc { // Supported scalability modes. Most applications should use the @@ -18,7 +24,7 @@ namespace webrtc { // This list of currently recognized modes is intended for the api boundary // between webrtc and injected encoders. Any application usage outside of // injected encoders is strongly discouraged. -enum class ScalabilityMode { +enum class ScalabilityMode : uint8_t { kL1T1, kL1T2, kL1T2h, @@ -48,5 +54,44 @@ enum class ScalabilityMode { kS3T3, }; +inline constexpr ScalabilityMode kAllScalabilityModes[] = { + // clang-format off + ScalabilityMode::kL1T1, + ScalabilityMode::kL1T2, + ScalabilityMode::kL1T2h, + ScalabilityMode::kL1T3, + ScalabilityMode::kL1T3h, + ScalabilityMode::kL2T1, + ScalabilityMode::kL2T1h, + ScalabilityMode::kL2T1_KEY, + ScalabilityMode::kL2T2, + ScalabilityMode::kL2T2h, + ScalabilityMode::kL2T2_KEY, + ScalabilityMode::kL2T2_KEY_SHIFT, + ScalabilityMode::kL2T3, + ScalabilityMode::kL2T3h, + ScalabilityMode::kL2T3_KEY, + ScalabilityMode::kL3T1, + ScalabilityMode::kL3T1h, + ScalabilityMode::kL3T1_KEY, + ScalabilityMode::kL3T2, + ScalabilityMode::kL3T2h, + ScalabilityMode::kL3T2_KEY, + ScalabilityMode::kL3T3, + ScalabilityMode::kL3T3h, + ScalabilityMode::kL3T3_KEY, + ScalabilityMode::kS2T1, + ScalabilityMode::kS2T3, + ScalabilityMode::kS3T3, + // clang-format on +}; + +inline constexpr size_t kScalabilityModeCount = + sizeof(kAllScalabilityModes) / sizeof(ScalabilityMode); + +RTC_EXPORT +absl::string_view ScalabilityModeToString(ScalabilityMode scalability_mode); + } // namespace webrtc + #endif // API_VIDEO_CODECS_SCALABILITY_MODE_H_ diff --git a/api/video_codecs/sdp_video_format.cc b/api/video_codecs/sdp_video_format.cc index 67c89135d0..3543806b51 100644 --- a/api/video_codecs/sdp_video_format.cc +++ b/api/video_codecs/sdp_video_format.cc @@ -70,6 +70,15 @@ SdpVideoFormat::SdpVideoFormat(const std::string& name, const Parameters& parameters) : name(name), parameters(parameters) {} +SdpVideoFormat::SdpVideoFormat( + const std::string& name, + const Parameters& parameters, + const absl::InlinedVector& + scalability_modes) + : name(name), + parameters(parameters), + scalability_modes(scalability_modes) {} + SdpVideoFormat::SdpVideoFormat(const SdpVideoFormat&) = default; SdpVideoFormat::SdpVideoFormat(SdpVideoFormat&&) = default; SdpVideoFormat& SdpVideoFormat::operator=(const SdpVideoFormat&) = default; @@ -80,9 +89,24 @@ SdpVideoFormat::~SdpVideoFormat() = default; std::string SdpVideoFormat::ToString() const { rtc::StringBuilder builder; builder << "Codec name: " << name << ", parameters: {"; - for (const auto& kv : parameters) + for (const auto& kv : parameters) { builder << " " << kv.first << "=" << kv.second; + } + builder << " }"; + if (!scalability_modes.empty()) { + builder << ", scalability_modes: ["; + bool first = true; + for (const auto scalability_mode : scalability_modes) { + if (first) { + first = false; + } else { + builder << ", "; + } + builder << ScalabilityModeToString(scalability_mode); + } + builder << "]"; + } return builder.str(); } @@ -105,7 +129,8 @@ bool SdpVideoFormat::IsCodecInList( } bool operator==(const SdpVideoFormat& a, const SdpVideoFormat& b) { - return a.name == b.name && a.parameters == b.parameters; + return a.name == b.name && a.parameters == b.parameters && + a.scalability_modes == b.scalability_modes; } } // namespace webrtc diff --git a/api/video_codecs/sdp_video_format.h b/api/video_codecs/sdp_video_format.h index a1e23f4f9c..850632eb9e 100644 --- a/api/video_codecs/sdp_video_format.h +++ b/api/video_codecs/sdp_video_format.h @@ -14,7 +14,9 @@ #include #include +#include "absl/container/inlined_vector.h" #include "api/array_view.h" +#include "api/video_codecs/scalability_mode.h" #include "rtc_base/system/rtc_export.h" namespace webrtc { @@ -26,6 +28,11 @@ struct RTC_EXPORT SdpVideoFormat { explicit SdpVideoFormat(const std::string& name); SdpVideoFormat(const std::string& name, const Parameters& parameters); + SdpVideoFormat( + const std::string& name, + const Parameters& parameters, + const absl::InlinedVector& + scalability_modes); SdpVideoFormat(const SdpVideoFormat&); SdpVideoFormat(SdpVideoFormat&&); SdpVideoFormat& operator=(const SdpVideoFormat&); @@ -51,6 +58,7 @@ struct RTC_EXPORT SdpVideoFormat { std::string name; Parameters parameters; + absl::InlinedVector scalability_modes; }; } // namespace webrtc diff --git a/api/video_codecs/test/video_encoder_factory_template_tests.cc b/api/video_codecs/test/video_encoder_factory_template_tests.cc index b9dfac9d9e..4c3d0cd24e 100644 --- a/api/video_codecs/test/video_encoder_factory_template_tests.cc +++ b/api/video_codecs/test/video_encoder_factory_template_tests.cc @@ -17,6 +17,7 @@ #include "test/gmock.h" #include "test/gtest.h" +using ::testing::Contains; using ::testing::Each; using ::testing::Eq; using ::testing::Field; @@ -125,9 +126,12 @@ TEST(VideoEncoderFactoryTemplate, TwoTemplateAdaptersCodecSupport) { TEST(VideoEncoderFactoryTemplate, LibvpxVp8) { VideoEncoderFactoryTemplate factory; - const SdpVideoFormat kVp8Sdp("VP8"); - EXPECT_THAT(factory.GetSupportedFormats(), UnorderedElementsAre(kVp8Sdp)); - EXPECT_THAT(factory.CreateVideoEncoder(kVp8Sdp), Ne(nullptr)); + auto formats = factory.GetSupportedFormats(); + EXPECT_THAT(formats.size(), 1); + EXPECT_THAT(formats[0], Field(&SdpVideoFormat::name, "VP8")); + EXPECT_THAT(formats[0], Field(&SdpVideoFormat::scalability_modes, + Contains(ScalabilityMode::kL1T3))); + EXPECT_THAT(factory.CreateVideoEncoder(formats[0]), Ne(nullptr)); } TEST(VideoEncoderFactoryTemplate, LibvpxVp9) { @@ -135,6 +139,8 @@ TEST(VideoEncoderFactoryTemplate, LibvpxVp9) { auto formats = factory.GetSupportedFormats(); EXPECT_THAT(formats, Not(IsEmpty())); EXPECT_THAT(formats, Each(Field(&SdpVideoFormat::name, "VP9"))); + EXPECT_THAT(formats, Each(Field(&SdpVideoFormat::scalability_modes, + Contains(ScalabilityMode::kL3T3_KEY)))); EXPECT_THAT(factory.CreateVideoEncoder(formats[0]), Ne(nullptr)); } @@ -146,15 +152,20 @@ TEST(VideoEncoderFactoryTemplate, OpenH264) { auto formats = factory.GetSupportedFormats(); EXPECT_THAT(formats, Not(IsEmpty())); EXPECT_THAT(formats, Each(Field(&SdpVideoFormat::name, "H264"))); + EXPECT_THAT(formats, Each(Field(&SdpVideoFormat::scalability_modes, + Contains(ScalabilityMode::kL1T3)))); EXPECT_THAT(factory.CreateVideoEncoder(formats[0]), Ne(nullptr)); } #endif // defined(WEBRTC_USE_H264) TEST(VideoEncoderFactoryTemplate, LibaomAv1) { VideoEncoderFactoryTemplate factory; - const SdpVideoFormat kAv1Sdp("AV1"); - EXPECT_THAT(factory.GetSupportedFormats(), UnorderedElementsAre(kAv1Sdp)); - EXPECT_THAT(factory.CreateVideoEncoder(kAv1Sdp), Ne(nullptr)); + auto formats = factory.GetSupportedFormats(); + EXPECT_THAT(formats.size(), 1); + EXPECT_THAT(formats[0], Field(&SdpVideoFormat::name, "AV1")); + EXPECT_THAT(formats[0], Field(&SdpVideoFormat::scalability_modes, + Contains(ScalabilityMode::kL3T3_KEY))); + EXPECT_THAT(factory.CreateVideoEncoder(formats[0]), Ne(nullptr)); } } // namespace diff --git a/api/video_codecs/video_encoder_factory_template_libaom_av1_adapter.h b/api/video_codecs/video_encoder_factory_template_libaom_av1_adapter.h index d71cc00253..ce273ffb82 100644 --- a/api/video_codecs/video_encoder_factory_template_libaom_av1_adapter.h +++ b/api/video_codecs/video_encoder_factory_template_libaom_av1_adapter.h @@ -14,13 +14,17 @@ #include #include +#include "absl/container/inlined_vector.h" #include "modules/video_coding/codecs/av1/av1_svc_config.h" #include "modules/video_coding/codecs/av1/libaom_av1_encoder.h" namespace webrtc { struct LibaomAv1EncoderTemplateAdapter { static std::vector SupportedFormats() { - return {SdpVideoFormat("AV1")}; + absl::InlinedVector + scalability_modes = LibaomAv1EncoderSupportedScalabilityModes(); + return { + SdpVideoFormat("AV1", SdpVideoFormat::Parameters(), scalability_modes)}; } static std::unique_ptr CreateEncoder( diff --git a/api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h b/api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h index c00c6d1250..d7d0cd48f1 100644 --- a/api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h +++ b/api/video_codecs/video_encoder_factory_template_libvpx_vp8_adapter.h @@ -14,13 +14,21 @@ #include #include +#include "absl/container/inlined_vector.h" #include "modules/video_coding/codecs/vp8/include/vp8.h" #include "modules/video_coding/codecs/vp8/vp8_scalability.h" namespace webrtc { struct LibvpxVp8EncoderTemplateAdapter { static std::vector SupportedFormats() { - return {SdpVideoFormat("VP8")}; + absl::InlinedVector + scalability_modes; + for (const auto scalability_mode : kVP8SupportedScalabilityModes) { + scalability_modes.push_back(scalability_mode); + } + + return { + SdpVideoFormat("VP8", SdpVideoFormat::Parameters(), scalability_modes)}; } static std::unique_ptr CreateEncoder( diff --git a/api/video_codecs/video_encoder_factory_template_libvpx_vp9_adapter.h b/api/video_codecs/video_encoder_factory_template_libvpx_vp9_adapter.h index 9947ec9740..c10fda4dc2 100644 --- a/api/video_codecs/video_encoder_factory_template_libvpx_vp9_adapter.h +++ b/api/video_codecs/video_encoder_factory_template_libvpx_vp9_adapter.h @@ -19,7 +19,7 @@ namespace webrtc { struct LibvpxVp9EncoderTemplateAdapter { static std::vector SupportedFormats() { - return SupportedVP9Codecs(); + return SupportedVP9Codecs(/*add_scalability_modes=*/true); } static std::unique_ptr CreateEncoder( diff --git a/api/video_codecs/video_encoder_factory_template_open_h264_adapter.h b/api/video_codecs/video_encoder_factory_template_open_h264_adapter.h index 9959617c62..5a9742a694 100644 --- a/api/video_codecs/video_encoder_factory_template_open_h264_adapter.h +++ b/api/video_codecs/video_encoder_factory_template_open_h264_adapter.h @@ -22,7 +22,7 @@ namespace webrtc { #if defined(WEBRTC_USE_H264) struct OpenH264EncoderTemplateAdapter { static std::vector SupportedFormats() { - return SupportedH264Codecs(); + return SupportedH264Codecs(/*add_scalability_modes=*/true); } static std::unique_ptr CreateEncoder( diff --git a/media/BUILD.gn b/media/BUILD.gn index 0481e3f0fe..e533c10f7d 100644 --- a/media/BUILD.gn +++ b/media/BUILD.gn @@ -102,6 +102,7 @@ rtc_library("rtc_media_base") { ] absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/container:inlined_vector", "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] diff --git a/media/base/codec.cc b/media/base/codec.cc index 4935e94b25..8204ef3344 100644 --- a/media/base/codec.cc +++ b/media/base/codec.cc @@ -304,6 +304,7 @@ VideoCodec::VideoCodec() : Codec() { VideoCodec::VideoCodec(const webrtc::SdpVideoFormat& c) : Codec(0 /* id */, c.name, kVideoCodecClockrate) { params = c.parameters; + scalability_modes = c.scalability_modes; } VideoCodec::VideoCodec(const VideoCodec& c) = default; diff --git a/media/base/codec.h b/media/base/codec.h index 73190223ac..f0ed25123c 100644 --- a/media/base/codec.h +++ b/media/base/codec.h @@ -16,6 +16,7 @@ #include #include +#include "absl/container/inlined_vector.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "api/field_trials_view.h" @@ -152,6 +153,8 @@ struct AudioCodec : public Codec { struct RTC_EXPORT VideoCodec : public Codec { absl::optional packetization; + absl::InlinedVector + scalability_modes; // Creates a codec with the given parameters. VideoCodec(int id, const std::string& name); diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 16b88421fc..6b74d2b9ea 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -500,6 +500,7 @@ rtc_library("webrtc_h264") { "//third_party/libyuv", ] absl_deps = [ + "//third_party/abseil-cpp/absl/container:inlined_vector", "//third_party/abseil-cpp/absl/strings", "//third_party/abseil-cpp/absl/types:optional", ] @@ -737,6 +738,7 @@ rtc_library("webrtc_vp9") { ] absl_deps = [ "//third_party/abseil-cpp/absl/algorithm:container", + "//third_party/abseil-cpp/absl/container:inlined_vector", "//third_party/abseil-cpp/absl/memory", "//third_party/abseil-cpp/absl/strings:strings", ] diff --git a/modules/video_coding/codecs/av1/BUILD.gn b/modules/video_coding/codecs/av1/BUILD.gn index 08a86e0c33..91f2cdeca1 100644 --- a/modules/video_coding/codecs/av1/BUILD.gn +++ b/modules/video_coding/codecs/av1/BUILD.gn @@ -23,6 +23,8 @@ rtc_library("av1_svc_config") { "../../svc:scalability_structures", "../../svc:scalable_video_controller", ] + + absl_deps = [ "//third_party/abseil-cpp/absl/container:inlined_vector" ] } rtc_library("dav1d_decoder") { diff --git a/modules/video_coding/codecs/av1/av1_svc_config.cc b/modules/video_coding/codecs/av1/av1_svc_config.cc index dd66de03aa..43dcf96ab7 100644 --- a/modules/video_coding/codecs/av1/av1_svc_config.cc +++ b/modules/video_coding/codecs/av1/av1_svc_config.cc @@ -36,6 +36,17 @@ absl::optional BuildScalabilityMode(int num_temporal_layers, } } // namespace +absl::InlinedVector +LibaomAv1EncoderSupportedScalabilityModes() { + absl::InlinedVector scalability_modes; + for (ScalabilityMode scalability_mode : kAllScalabilityModes) { + if (ScalabilityStructureConfig(scalability_mode) != absl::nullopt) { + scalability_modes.push_back(scalability_mode); + } + } + return scalability_modes; +} + bool LibaomAv1EncoderSupportsScalabilityMode(ScalabilityMode scalability_mode) { // For libaom AV1, the scalability mode is supported if we can create the // scalability structure. diff --git a/modules/video_coding/codecs/av1/av1_svc_config.h b/modules/video_coding/codecs/av1/av1_svc_config.h index 56cc2b3e9e..05b886b9f4 100644 --- a/modules/video_coding/codecs/av1/av1_svc_config.h +++ b/modules/video_coding/codecs/av1/av1_svc_config.h @@ -10,10 +10,16 @@ #ifndef MODULES_VIDEO_CODING_CODECS_AV1_AV1_SVC_CONFIG_H_ #define MODULES_VIDEO_CODING_CODECS_AV1_AV1_SVC_CONFIG_H_ +#include + +#include "absl/container/inlined_vector.h" #include "api/video_codecs/video_codec.h" namespace webrtc { +absl::InlinedVector +LibaomAv1EncoderSupportedScalabilityModes(); + bool LibaomAv1EncoderSupportsScalabilityMode(ScalabilityMode scalability_mode); // Fills `video_codec.spatialLayers` using other members. diff --git a/modules/video_coding/codecs/h264/h264.cc b/modules/video_coding/codecs/h264/h264.cc index b77cf0d149..23580d7a4a 100644 --- a/modules/video_coding/codecs/h264/h264.cc +++ b/modules/video_coding/codecs/h264/h264.cc @@ -14,6 +14,7 @@ #include #include +#include "absl/container/inlined_vector.h" #include "absl/types/optional.h" #include "api/video_codecs/sdp_video_format.h" #include "media/base/media_constants.h" @@ -51,15 +52,23 @@ constexpr ScalabilityMode kSupportedScalabilityModes[] = { SdpVideoFormat CreateH264Format(H264Profile profile, H264Level level, - const std::string& packetization_mode) { + const std::string& packetization_mode, + bool add_scalability_modes) { const absl::optional profile_string = H264ProfileLevelIdToString(H264ProfileLevelId(profile, level)); RTC_CHECK(profile_string); + absl::InlinedVector scalability_modes; + if (add_scalability_modes) { + for (const auto scalability_mode : kSupportedScalabilityModes) { + scalability_modes.push_back(scalability_mode); + } + } return SdpVideoFormat( cricket::kH264CodecName, {{cricket::kH264FmtpProfileLevelId, *profile_string}, {cricket::kH264FmtpLevelAsymmetryAllowed, "1"}, - {cricket::kH264FmtpPacketizationMode, packetization_mode}}); + {cricket::kH264FmtpPacketizationMode, packetization_mode}}, + scalability_modes); } void DisableRtcUseH264() { @@ -68,7 +77,7 @@ void DisableRtcUseH264() { #endif } -std::vector SupportedH264Codecs() { +std::vector SupportedH264Codecs(bool add_scalability_modes) { TRACE_EVENT0("webrtc", __func__); if (!IsH264CodecSupported()) return std::vector(); @@ -81,17 +90,18 @@ std::vector SupportedH264Codecs() { // // We support both packetization modes 0 (mandatory) and 1 (optional, // preferred). - return { - CreateH264Format(H264Profile::kProfileBaseline, H264Level::kLevel3_1, - "1"), - CreateH264Format(H264Profile::kProfileBaseline, H264Level::kLevel3_1, - "0"), - CreateH264Format(H264Profile::kProfileConstrainedBaseline, - H264Level::kLevel3_1, "1"), - CreateH264Format(H264Profile::kProfileConstrainedBaseline, - H264Level::kLevel3_1, "0"), - CreateH264Format(H264Profile::kProfileMain, H264Level::kLevel3_1, "1"), - CreateH264Format(H264Profile::kProfileMain, H264Level::kLevel3_1, "0")}; + return {CreateH264Format(H264Profile::kProfileBaseline, H264Level::kLevel3_1, + "1", add_scalability_modes), + CreateH264Format(H264Profile::kProfileBaseline, H264Level::kLevel3_1, + "0", add_scalability_modes), + CreateH264Format(H264Profile::kProfileConstrainedBaseline, + H264Level::kLevel3_1, "1", add_scalability_modes), + CreateH264Format(H264Profile::kProfileConstrainedBaseline, + H264Level::kLevel3_1, "0", add_scalability_modes), + CreateH264Format(H264Profile::kProfileMain, H264Level::kLevel3_1, "1", + add_scalability_modes), + CreateH264Format(H264Profile::kProfileMain, H264Level::kLevel3_1, "0", + add_scalability_modes)}; } std::vector SupportedH264DecoderCodecs() { diff --git a/modules/video_coding/codecs/h264/include/h264.h b/modules/video_coding/codecs/h264/include/h264.h index ff7e16851b..2635b53842 100644 --- a/modules/video_coding/codecs/h264/include/h264.h +++ b/modules/video_coding/codecs/h264/include/h264.h @@ -30,7 +30,8 @@ struct SdpVideoFormat; RTC_EXPORT SdpVideoFormat CreateH264Format(H264Profile profile, H264Level level, - const std::string& packetization_mode); + const std::string& packetization_mode, + bool add_scalability_modes = false); // Set to disable the H.264 encoder/decoder implementations that are provided if // `rtc_use_h264` build flag is true (if false, this function does nothing). @@ -40,7 +41,8 @@ RTC_EXPORT void DisableRtcUseH264(); // Returns a vector with all supported internal H264 encode profiles that we can // negotiate in SDP, in order of preference. -std::vector SupportedH264Codecs(); +std::vector SupportedH264Codecs( + bool add_scalability_modes = false); // Returns a vector with all supported internal H264 decode profiles that we can // negotiate in SDP, in order of preference. This will be available for receive diff --git a/modules/video_coding/codecs/vp8/vp8_scalability.cc b/modules/video_coding/codecs/vp8/vp8_scalability.cc index df15be0127..9c7495ddf7 100644 --- a/modules/video_coding/codecs/vp8/vp8_scalability.cc +++ b/modules/video_coding/codecs/vp8/vp8_scalability.cc @@ -13,9 +13,7 @@ namespace webrtc { bool VP8SupportsScalabilityMode(ScalabilityMode scalability_mode) { - constexpr ScalabilityMode kSupportedScalabilityModes[] = { - ScalabilityMode::kL1T1, ScalabilityMode::kL1T2, ScalabilityMode::kL1T3}; - for (const auto& entry : kSupportedScalabilityModes) { + for (const auto& entry : kVP8SupportedScalabilityModes) { if (entry == scalability_mode) { return true; } diff --git a/modules/video_coding/codecs/vp8/vp8_scalability.h b/modules/video_coding/codecs/vp8/vp8_scalability.h index 11f018a564..923f159118 100644 --- a/modules/video_coding/codecs/vp8/vp8_scalability.h +++ b/modules/video_coding/codecs/vp8/vp8_scalability.h @@ -15,6 +15,8 @@ namespace webrtc { +inline constexpr ScalabilityMode kVP8SupportedScalabilityModes[] = { + ScalabilityMode::kL1T1, ScalabilityMode::kL1T2, ScalabilityMode::kL1T3}; bool VP8SupportsScalabilityMode(ScalabilityMode scalability_mode); } // namespace webrtc diff --git a/modules/video_coding/codecs/vp9/include/vp9.h b/modules/video_coding/codecs/vp9/include/vp9.h index 9ef36b00ef..79d403ded3 100644 --- a/modules/video_coding/codecs/vp9/include/vp9.h +++ b/modules/video_coding/codecs/vp9/include/vp9.h @@ -24,7 +24,8 @@ namespace webrtc { // Returns a vector with all supported internal VP9 profiles that we can // negotiate in SDP, in order of preference. -std::vector SupportedVP9Codecs(); +std::vector SupportedVP9Codecs( + bool add_scalability_modes = false); // Returns a vector with all supported internal VP9 decode profiles in order of // preference. These will be availble for receive-only connections. diff --git a/modules/video_coding/codecs/vp9/vp9.cc b/modules/video_coding/codecs/vp9/vp9.cc index 7a7a7d6c8f..222e57b6ba 100644 --- a/modules/video_coding/codecs/vp9/vp9.cc +++ b/modules/video_coding/codecs/vp9/vp9.cc @@ -12,6 +12,7 @@ #include +#include "absl/container/inlined_vector.h" #include "api/transport/field_trial_based_config.h" #include "api/video_codecs/scalability_mode.h" #include "api/video_codecs/sdp_video_format.h" @@ -26,7 +27,7 @@ namespace webrtc { -std::vector SupportedVP9Codecs() { +std::vector SupportedVP9Codecs(bool add_scalability_modes) { #ifdef RTC_ENABLE_VP9 // Profile 2 might not be available on some platforms until // https://bugs.chromium.org/p/webm/issues/detail?id=1544 is solved. @@ -36,13 +37,23 @@ std::vector SupportedVP9Codecs() { (vpx_codec_get_caps(vpx_codec_vp9_dx()) & VPX_CODEC_CAP_HIGHBITDEPTH) != 0; + absl::InlinedVector scalability_modes; + if (add_scalability_modes) { + for (const auto scalability_mode : kAllScalabilityModes) { + if (ScalabilityStructureConfig(scalability_mode).has_value()) { + scalability_modes.push_back(scalability_mode); + } + } + } std::vector supported_formats{SdpVideoFormat( cricket::kVp9CodecName, - {{kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile0)}})}; + {{kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile0)}}, + scalability_modes)}; if (vpx_supports_high_bit_depth) { supported_formats.push_back(SdpVideoFormat( cricket::kVp9CodecName, - {{kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile2)}})); + {{kVP9FmtpProfileId, VP9ProfileToString(VP9Profile::kProfile2)}}, + scalability_modes)); } return supported_formats; diff --git a/modules/video_coding/svc/scalability_mode_util.cc b/modules/video_coding/svc/scalability_mode_util.cc index 510c9fd32c..addcdd1342 100644 --- a/modules/video_coding/svc/scalability_mode_util.cc +++ b/modules/video_coding/svc/scalability_mode_util.cc @@ -83,66 +83,6 @@ absl::optional ScalabilityModeFromString( return absl::nullopt; } -absl::string_view ScalabilityModeToString(ScalabilityMode scalability_mode) { - switch (scalability_mode) { - case ScalabilityMode::kL1T1: - return "L1T1"; - case ScalabilityMode::kL1T2: - return "L1T2"; - case ScalabilityMode::kL1T2h: - return "L1T2h"; - case ScalabilityMode::kL1T3: - return "L1T3"; - case ScalabilityMode::kL1T3h: - return "L1T3h"; - case ScalabilityMode::kL2T1: - return "L2T1"; - case ScalabilityMode::kL2T1h: - return "L2T1h"; - case ScalabilityMode::kL2T1_KEY: - return "L2T1_KEY"; - case ScalabilityMode::kL2T2: - return "L2T2"; - case ScalabilityMode::kL2T2h: - return "L2T2h"; - case ScalabilityMode::kL2T2_KEY: - return "L2T2_KEY"; - case ScalabilityMode::kL2T2_KEY_SHIFT: - return "L2T2_KEY_SHIFT"; - case ScalabilityMode::kL2T3: - return "L2T3"; - case ScalabilityMode::kL2T3h: - return "L2T3h"; - case ScalabilityMode::kL2T3_KEY: - return "L2T3_KEY"; - case ScalabilityMode::kL3T1: - return "L3T1"; - case ScalabilityMode::kL3T1h: - return "L3T1h"; - case ScalabilityMode::kL3T1_KEY: - return "L3T1_KEY"; - case ScalabilityMode::kL3T2: - return "L3T2"; - case ScalabilityMode::kL3T2h: - return "L3T2h"; - case ScalabilityMode::kL3T2_KEY: - return "L3T2_KEY"; - case ScalabilityMode::kL3T3: - return "L3T3"; - case ScalabilityMode::kL3T3h: - return "L3T3h"; - case ScalabilityMode::kL3T3_KEY: - return "L3T3_KEY"; - case ScalabilityMode::kS2T1: - return "S2T1"; - case ScalabilityMode::kS2T3: - return "S2T3"; - case ScalabilityMode::kS3T3: - return "S3T3"; - } - RTC_CHECK_NOTREACHED(); -} - InterLayerPredMode ScalabilityModeToInterLayerPredMode( ScalabilityMode scalability_mode) { switch (scalability_mode) { diff --git a/modules/video_coding/svc/scalability_mode_util.h b/modules/video_coding/svc/scalability_mode_util.h index c543c0c15a..e9308bde33 100644 --- a/modules/video_coding/svc/scalability_mode_util.h +++ b/modules/video_coding/svc/scalability_mode_util.h @@ -21,8 +21,6 @@ namespace webrtc { absl::optional ScalabilityModeFromString( absl::string_view scalability_mode_string); -absl::string_view ScalabilityModeToString(ScalabilityMode scalability_mode); - InterLayerPredMode ScalabilityModeToInterLayerPredMode( ScalabilityMode scalability_mode); diff --git a/pc/peer_connection_factory_unittest.cc b/pc/peer_connection_factory_unittest.cc index 7f71c761da..b3e9e1ebbe 100644 --- a/pc/peer_connection_factory_unittest.cc +++ b/pc/peer_connection_factory_unittest.cc @@ -61,6 +61,7 @@ using ::testing::AtLeast; using ::testing::InvokeWithoutArgs; using ::testing::NiceMock; using ::testing::Return; +using ::testing::UnorderedElementsAre; namespace { @@ -181,10 +182,49 @@ class PeerConnectionFactoryTest : public ::testing::Test { EXPECT_GT(codec.num_channels, 0); } - void VerifyVideoCodecCapability(const webrtc::RtpCodecCapability& codec) { + void VerifyVideoCodecCapability(const webrtc::RtpCodecCapability& codec, + bool sender) { EXPECT_EQ(codec.kind, cricket::MEDIA_TYPE_VIDEO); EXPECT_FALSE(codec.name.empty()); EXPECT_GT(codec.clock_rate, 0); + if (sender) { + if (codec.name == "VP8" || codec.name == "H264") { + EXPECT_THAT(codec.scalability_modes, + UnorderedElementsAre(webrtc::ScalabilityMode::kL1T1, + webrtc::ScalabilityMode::kL1T2, + webrtc::ScalabilityMode::kL1T3)) + << "Codec: " << codec.name; + } else if (codec.name == "VP9" || codec.name == "AV1") { + EXPECT_THAT( + codec.scalability_modes, + UnorderedElementsAre( + // clang-format off + webrtc::ScalabilityMode::kL1T1, + webrtc::ScalabilityMode::kL1T2, + webrtc::ScalabilityMode::kL1T3, + webrtc::ScalabilityMode::kL2T1, + webrtc::ScalabilityMode::kL2T1h, + webrtc::ScalabilityMode::kL2T1_KEY, + webrtc::ScalabilityMode::kL2T2, + webrtc::ScalabilityMode::kL2T2_KEY, + webrtc::ScalabilityMode::kL2T2_KEY_SHIFT, + webrtc::ScalabilityMode::kL2T3, + webrtc::ScalabilityMode::kL2T3_KEY, + webrtc::ScalabilityMode::kL3T1, + webrtc::ScalabilityMode::kL3T3, + webrtc::ScalabilityMode::kL3T3_KEY, + webrtc::ScalabilityMode::kS2T1, + webrtc::ScalabilityMode::kS2T3, + webrtc::ScalabilityMode::kS3T3) + // clang-format on + ) + << "Codec: " << codec.name; + } else { + EXPECT_TRUE(codec.scalability_modes.empty()); + } + } else { + EXPECT_TRUE(codec.scalability_modes.empty()); + } } std::unique_ptr socket_server_; @@ -251,7 +291,7 @@ TEST_F(PeerConnectionFactoryTest, CheckRtpSenderVideoCapabilities) { factory_->GetRtpSenderCapabilities(cricket::MEDIA_TYPE_VIDEO); EXPECT_FALSE(video_capabilities.codecs.empty()); for (const auto& codec : video_capabilities.codecs) { - VerifyVideoCodecCapability(codec); + VerifyVideoCodecCapability(codec, true); } EXPECT_FALSE(video_capabilities.header_extensions.empty()); for (const auto& header_extension : video_capabilities.header_extensions) { @@ -284,7 +324,7 @@ TEST_F(PeerConnectionFactoryTest, CheckRtpReceiverVideoCapabilities) { factory_->GetRtpReceiverCapabilities(cricket::MEDIA_TYPE_VIDEO); EXPECT_FALSE(video_capabilities.codecs.empty()); for (const auto& codec : video_capabilities.codecs) { - VerifyVideoCodecCapability(codec); + VerifyVideoCodecCapability(codec, false); } EXPECT_FALSE(video_capabilities.header_extensions.empty()); for (const auto& header_extension : video_capabilities.header_extensions) { diff --git a/pc/rtp_parameters_conversion.cc b/pc/rtp_parameters_conversion.cc index afba4bc94f..196cb79ea5 100644 --- a/pc/rtp_parameters_conversion.cc +++ b/pc/rtp_parameters_conversion.cc @@ -308,7 +308,18 @@ void ToRtpCodecCapabilityTypeSpecific( template <> void ToRtpCodecCapabilityTypeSpecific( const cricket::VideoCodec& cricket_codec, - RtpCodecCapability* codec) {} + RtpCodecCapability* codec) { + if (cricket_codec.scalability_modes.empty() || + (cricket_codec.scalability_modes.size() == 1 && + cricket_codec.scalability_modes[0] == ScalabilityMode::kL1T1)) { + // https://w3c.github.io/webrtc-svc/#dom-rtcrtpcodeccapability-scalabilitymodes + // If a codec does not support encoding of scalability modes other than + // "L1T1", then the scalabilityModes member is not provided. + return; + } + + codec->scalability_modes = cricket_codec.scalability_modes; +} template RtpCodecCapability ToRtpCodecCapability(const C& cricket_codec) {