diff --git a/api/video_codecs/scalability_mode.cc b/api/video_codecs/scalability_mode.cc index ab26915c98..c449b4217e 100644 --- a/api/video_codecs/scalability_mode.cc +++ b/api/video_codecs/scalability_mode.cc @@ -62,16 +62,28 @@ absl::string_view ScalabilityModeToString(ScalabilityMode scalability_mode) { return "L3T3_KEY"; case ScalabilityMode::kS2T1: return "S2T1"; + case ScalabilityMode::kS2T1h: + return "S2T1h"; case ScalabilityMode::kS2T2: return "S2T2"; + case ScalabilityMode::kS2T2h: + return "S2T2h"; case ScalabilityMode::kS2T3: return "S2T3"; + case ScalabilityMode::kS2T3h: + return "S2T3h"; case ScalabilityMode::kS3T1: return "S3T1"; + case ScalabilityMode::kS3T1h: + return "S3T1h"; case ScalabilityMode::kS3T2: return "S3T2"; + case ScalabilityMode::kS3T2h: + return "S3T2h"; case ScalabilityMode::kS3T3: return "S3T3"; + case ScalabilityMode::kS3T3h: + return "S3T3h"; } RTC_CHECK_NOTREACHED(); } diff --git a/api/video_codecs/scalability_mode.h b/api/video_codecs/scalability_mode.h index 09f564e8e6..b26f32eb22 100644 --- a/api/video_codecs/scalability_mode.h +++ b/api/video_codecs/scalability_mode.h @@ -48,11 +48,17 @@ enum class ScalabilityMode : uint8_t { kL3T3h, kL3T3_KEY, kS2T1, + kS2T1h, kS2T2, + kS2T2h, kS2T3, + kS2T3h, kS3T1, + kS3T1h, kS3T2, + kS3T2h, kS3T3, + kS3T3h, }; inline constexpr ScalabilityMode kAllScalabilityModes[] = { @@ -80,11 +86,17 @@ inline constexpr ScalabilityMode kAllScalabilityModes[] = { ScalabilityMode::kL3T3h, ScalabilityMode::kL3T3_KEY, ScalabilityMode::kS2T1, + ScalabilityMode::kS2T1h, ScalabilityMode::kS2T2, + ScalabilityMode::kS2T2h, ScalabilityMode::kS2T3, + ScalabilityMode::kS2T3h, ScalabilityMode::kS3T1, + ScalabilityMode::kS3T1h, ScalabilityMode::kS3T2, + ScalabilityMode::kS3T2h, ScalabilityMode::kS3T3, + ScalabilityMode::kS3T3h, // clang-format on }; diff --git a/modules/video_coding/svc/create_scalability_structure.cc b/modules/video_coding/svc/create_scalability_structure.cc index dd3225cbf4..fbcd27b139 100644 --- a/modules/video_coding/svc/create_scalability_structure.cc +++ b/modules/video_coding/svc/create_scalability_structure.cc @@ -148,6 +148,13 @@ constexpr ScalableVideoController::StreamLayersConfig kConfigS2T1 = { {1, 1}, {2, 1}}; +constexpr ScalableVideoController::StreamLayersConfig kConfigS2T1h = { + /*num_spatial_layers=*/2, + /*num_temporal_layers=*/1, + /*uses_reference_scaling=*/false, + {2, 1}, + {3, 1}}; + constexpr ScalableVideoController::StreamLayersConfig kConfigS2T2 = { /*num_spatial_layers=*/2, /*num_temporal_layers=*/2, @@ -155,6 +162,13 @@ constexpr ScalableVideoController::StreamLayersConfig kConfigS2T2 = { {1, 1}, {2, 1}}; +constexpr ScalableVideoController::StreamLayersConfig kConfigS2T2h = { + /*num_spatial_layers=*/2, + /*num_temporal_layers=*/2, + /*uses_reference_scaling=*/false, + {2, 1}, + {3, 1}}; + constexpr ScalableVideoController::StreamLayersConfig kConfigS2T3 = { /*num_spatial_layers=*/2, /*num_temporal_layers=*/3, @@ -162,6 +176,13 @@ constexpr ScalableVideoController::StreamLayersConfig kConfigS2T3 = { {1, 1}, {2, 1}}; +constexpr ScalableVideoController::StreamLayersConfig kConfigS2T3h = { + /*num_spatial_layers=*/2, + /*num_temporal_layers=*/3, + /*uses_reference_scaling=*/false, + {2, 1}, + {3, 1}}; + constexpr ScalableVideoController::StreamLayersConfig kConfigS3T1 = { /*num_spatial_layers=*/3, /*num_temporal_layers=*/1, @@ -169,6 +190,13 @@ constexpr ScalableVideoController::StreamLayersConfig kConfigS3T1 = { {1, 1, 1}, {4, 2, 1}}; +constexpr ScalableVideoController::StreamLayersConfig kConfigS3T1h = { + /*num_spatial_layers=*/3, + /*num_temporal_layers=*/1, + /*uses_reference_scaling=*/false, + {4, 2, 1}, + {9, 3, 1}}; + constexpr ScalableVideoController::StreamLayersConfig kConfigS3T2 = { /*num_spatial_layers=*/3, /*num_temporal_layers=*/2, @@ -176,6 +204,13 @@ constexpr ScalableVideoController::StreamLayersConfig kConfigS3T2 = { {1, 1, 1}, {4, 2, 1}}; +constexpr ScalableVideoController::StreamLayersConfig kConfigS3T2h = { + /*num_spatial_layers=*/3, + /*num_temporal_layers=*/2, + /*uses_reference_scaling=*/false, + {4, 2, 1}, + {9, 3, 1}}; + constexpr ScalableVideoController::StreamLayersConfig kConfigS3T3 = { /*num_spatial_layers=*/3, /*num_temporal_layers=*/3, @@ -183,6 +218,13 @@ constexpr ScalableVideoController::StreamLayersConfig kConfigS3T3 = { {1, 1, 1}, {4, 2, 1}}; +constexpr ScalableVideoController::StreamLayersConfig kConfigS3T3h = { + /*num_spatial_layers=*/3, + /*num_temporal_layers=*/3, + /*uses_reference_scaling=*/false, + {4, 2, 1}, + {9, 3, 1}}; + constexpr NamedStructureFactory kFactories[] = { {ScalabilityMode::kL1T1, Create, kConfigL1T1}, @@ -215,11 +257,17 @@ constexpr NamedStructureFactory kFactories[] = { {ScalabilityMode::kL3T3_KEY, Create, kConfigL3T3}, {ScalabilityMode::kS2T1, Create, kConfigS2T1}, + {ScalabilityMode::kS2T1h, CreateH, kConfigS2T1h}, {ScalabilityMode::kS2T2, Create, kConfigS2T2}, + {ScalabilityMode::kS2T2h, CreateH, kConfigS2T2h}, {ScalabilityMode::kS2T3, Create, kConfigS2T3}, + {ScalabilityMode::kS2T3h, CreateH, kConfigS2T3h}, {ScalabilityMode::kS3T1, Create, kConfigS3T1}, + {ScalabilityMode::kS3T1h, CreateH, kConfigS3T1h}, {ScalabilityMode::kS3T2, Create, kConfigS3T2}, + {ScalabilityMode::kS3T2h, CreateH, kConfigS3T2h}, {ScalabilityMode::kS3T3, Create, kConfigS3T3}, + {ScalabilityMode::kS3T3h, CreateH, kConfigS3T3h}, }; } // namespace diff --git a/modules/video_coding/svc/scalability_mode_util.cc b/modules/video_coding/svc/scalability_mode_util.cc index 070dcfe15c..39a4f1fd1e 100644 --- a/modules/video_coding/svc/scalability_mode_util.cc +++ b/modules/video_coding/svc/scalability_mode_util.cc @@ -71,16 +71,28 @@ absl::optional ScalabilityModeFromString( if (mode_string == "S2T1") return ScalabilityMode::kS2T1; + if (mode_string == "S2T1h") + return ScalabilityMode::kS2T1h; if (mode_string == "S2T2") return ScalabilityMode::kS2T2; + if (mode_string == "S2T2h") + return ScalabilityMode::kS2T2h; if (mode_string == "S2T3") return ScalabilityMode::kS2T3; + if (mode_string == "S2T3h") + return ScalabilityMode::kS2T3h; if (mode_string == "S3T1") return ScalabilityMode::kS3T1; + if (mode_string == "S3T1h") + return ScalabilityMode::kS3T1h; if (mode_string == "S3T2") return ScalabilityMode::kS3T2; + if (mode_string == "S3T2h") + return ScalabilityMode::kS3T2h; if (mode_string == "S3T3") return ScalabilityMode::kS3T3; + if (mode_string == "S3T3h") + return ScalabilityMode::kS3T3h; return absl::nullopt; } @@ -123,11 +135,17 @@ InterLayerPredMode ScalabilityModeToInterLayerPredMode( case ScalabilityMode::kL3T3_KEY: return InterLayerPredMode::kOnKeyPic; case ScalabilityMode::kS2T1: + case ScalabilityMode::kS2T1h: case ScalabilityMode::kS2T2: + case ScalabilityMode::kS2T2h: case ScalabilityMode::kS2T3: + case ScalabilityMode::kS2T3h: case ScalabilityMode::kS3T1: + case ScalabilityMode::kS3T1h: case ScalabilityMode::kS3T2: + case ScalabilityMode::kS3T2h: case ScalabilityMode::kS3T3: + case ScalabilityMode::kS3T3h: return InterLayerPredMode::kOff; } RTC_CHECK_NOTREACHED(); @@ -161,12 +179,18 @@ int ScalabilityModeToNumSpatialLayers(ScalabilityMode scalability_mode) { case ScalabilityMode::kL3T3_KEY: return 3; case ScalabilityMode::kS2T1: + case ScalabilityMode::kS2T1h: case ScalabilityMode::kS2T2: + case ScalabilityMode::kS2T2h: case ScalabilityMode::kS2T3: + case ScalabilityMode::kS2T3h: return 2; case ScalabilityMode::kS3T1: + case ScalabilityMode::kS3T1h: case ScalabilityMode::kS3T2: + case ScalabilityMode::kS3T2h: case ScalabilityMode::kS3T3: + case ScalabilityMode::kS3T3h: return 3; } RTC_CHECK_NOTREACHED(); @@ -206,13 +230,19 @@ int ScalabilityModeToNumTemporalLayers(ScalabilityMode scalability_mode) { case ScalabilityMode::kL3T3_KEY: return 3; case ScalabilityMode::kS2T1: + case ScalabilityMode::kS2T1h: case ScalabilityMode::kS3T1: + case ScalabilityMode::kS3T1h: return 1; case ScalabilityMode::kS2T2: + case ScalabilityMode::kS2T2h: case ScalabilityMode::kS3T2: + case ScalabilityMode::kS3T2h: return 2; case ScalabilityMode::kS2T3: + case ScalabilityMode::kS2T3h: case ScalabilityMode::kS3T3: + case ScalabilityMode::kS3T3h: return 3; } RTC_CHECK_NOTREACHED(); @@ -251,6 +281,12 @@ absl::optional ScalabilityModeToResolutionRatio( case ScalabilityMode::kL3T1h: case ScalabilityMode::kL3T2h: case ScalabilityMode::kL3T3h: + case ScalabilityMode::kS2T1h: + case ScalabilityMode::kS2T2h: + case ScalabilityMode::kS2T3h: + case ScalabilityMode::kS3T1h: + case ScalabilityMode::kS3T2h: + case ScalabilityMode::kS3T3h: return ScalabilityModeResolutionRatio::kThreeToTwo; } RTC_CHECK_NOTREACHED(); diff --git a/modules/video_coding/svc/scalability_structure_simulcast.cc b/modules/video_coding/svc/scalability_structure_simulcast.cc index 010c461dc2..54e27fda5c 100644 --- a/modules/video_coding/svc/scalability_structure_simulcast.cc +++ b/modules/video_coding/svc/scalability_structure_simulcast.cc @@ -43,9 +43,11 @@ constexpr int ScalabilityStructureSimulcast::kMaxNumTemporalLayers; ScalabilityStructureSimulcast::ScalabilityStructureSimulcast( int num_spatial_layers, - int num_temporal_layers) + int num_temporal_layers, + ScalingFactor resolution_factor) : num_spatial_layers_(num_spatial_layers), num_temporal_layers_(num_temporal_layers), + resolution_factor_(resolution_factor), active_decode_targets_( (uint32_t{1} << (num_spatial_layers * num_temporal_layers)) - 1) { RTC_DCHECK_LE(num_spatial_layers, kMaxNumSpatialLayers); @@ -62,8 +64,10 @@ ScalabilityStructureSimulcast::StreamConfig() const { result.scaling_factor_num[num_spatial_layers_ - 1] = 1; result.scaling_factor_den[num_spatial_layers_ - 1] = 1; for (int sid = num_spatial_layers_ - 1; sid > 0; --sid) { - result.scaling_factor_num[sid - 1] = 1; - result.scaling_factor_den[sid - 1] = 2 * result.scaling_factor_den[sid]; + result.scaling_factor_num[sid - 1] = + resolution_factor_.num * result.scaling_factor_num[sid]; + result.scaling_factor_den[sid - 1] = + resolution_factor_.den * result.scaling_factor_den[sid]; } result.uses_reference_scaling = false; return result; diff --git a/modules/video_coding/svc/scalability_structure_simulcast.h b/modules/video_coding/svc/scalability_structure_simulcast.h index 7438173788..99be9f0d58 100644 --- a/modules/video_coding/svc/scalability_structure_simulcast.h +++ b/modules/video_coding/svc/scalability_structure_simulcast.h @@ -23,8 +23,13 @@ namespace webrtc { // same temporal layering. class ScalabilityStructureSimulcast : public ScalableVideoController { public: + struct ScalingFactor { + int num = 1; + int den = 2; + }; ScalabilityStructureSimulcast(int num_spatial_layers, - int num_temporal_layers); + int num_temporal_layers, + ScalingFactor resolution_factor); ~ScalabilityStructureSimulcast() override; StreamLayersConfig StreamConfig() const override; @@ -58,6 +63,7 @@ class ScalabilityStructureSimulcast : public ScalableVideoController { const int num_spatial_layers_; const int num_temporal_layers_; + const ScalingFactor resolution_factor_; FramePattern last_pattern_ = kNone; std::bitset can_reference_t0_frame_for_spatial_id_ = 0; @@ -70,7 +76,8 @@ class ScalabilityStructureSimulcast : public ScalableVideoController { // S0 0--0--0- class ScalabilityStructureS2T1 : public ScalabilityStructureSimulcast { public: - ScalabilityStructureS2T1() : ScalabilityStructureSimulcast(2, 1) {} + explicit ScalabilityStructureS2T1(ScalingFactor resolution_factor = {}) + : ScalabilityStructureSimulcast(2, 1, resolution_factor) {} ~ScalabilityStructureS2T1() override = default; FrameDependencyStructure DependencyStructure() const override; @@ -78,7 +85,8 @@ class ScalabilityStructureS2T1 : public ScalabilityStructureSimulcast { class ScalabilityStructureS2T2 : public ScalabilityStructureSimulcast { public: - ScalabilityStructureS2T2() : ScalabilityStructureSimulcast(2, 2) {} + explicit ScalabilityStructureS2T2(ScalingFactor resolution_factor = {}) + : ScalabilityStructureSimulcast(2, 2, resolution_factor) {} ~ScalabilityStructureS2T2() override = default; FrameDependencyStructure DependencyStructure() const override; @@ -98,7 +106,8 @@ class ScalabilityStructureS2T2 : public ScalabilityStructureSimulcast { // Time-> 0 1 2 3 4 class ScalabilityStructureS2T3 : public ScalabilityStructureSimulcast { public: - ScalabilityStructureS2T3() : ScalabilityStructureSimulcast(2, 3) {} + explicit ScalabilityStructureS2T3(ScalingFactor resolution_factor = {}) + : ScalabilityStructureSimulcast(2, 3, resolution_factor) {} ~ScalabilityStructureS2T3() override = default; FrameDependencyStructure DependencyStructure() const override; @@ -106,7 +115,8 @@ class ScalabilityStructureS2T3 : public ScalabilityStructureSimulcast { class ScalabilityStructureS3T1 : public ScalabilityStructureSimulcast { public: - ScalabilityStructureS3T1() : ScalabilityStructureSimulcast(3, 1) {} + explicit ScalabilityStructureS3T1(ScalingFactor resolution_factor = {}) + : ScalabilityStructureSimulcast(3, 1, resolution_factor) {} ~ScalabilityStructureS3T1() override = default; FrameDependencyStructure DependencyStructure() const override; @@ -114,7 +124,8 @@ class ScalabilityStructureS3T1 : public ScalabilityStructureSimulcast { class ScalabilityStructureS3T2 : public ScalabilityStructureSimulcast { public: - ScalabilityStructureS3T2() : ScalabilityStructureSimulcast(3, 2) {} + explicit ScalabilityStructureS3T2(ScalingFactor resolution_factor = {}) + : ScalabilityStructureSimulcast(3, 2, resolution_factor) {} ~ScalabilityStructureS3T2() override = default; FrameDependencyStructure DependencyStructure() const override; @@ -122,7 +133,8 @@ class ScalabilityStructureS3T2 : public ScalabilityStructureSimulcast { class ScalabilityStructureS3T3 : public ScalabilityStructureSimulcast { public: - ScalabilityStructureS3T3() : ScalabilityStructureSimulcast(3, 3) {} + explicit ScalabilityStructureS3T3(ScalingFactor resolution_factor = {}) + : ScalabilityStructureSimulcast(3, 3, resolution_factor) {} ~ScalabilityStructureS3T3() override = default; FrameDependencyStructure DependencyStructure() const override; diff --git a/pc/peer_connection_factory_unittest.cc b/pc/peer_connection_factory_unittest.cc index fdbc415fba..2251990963 100644 --- a/pc/peer_connection_factory_unittest.cc +++ b/pc/peer_connection_factory_unittest.cc @@ -222,11 +222,17 @@ class PeerConnectionFactoryTest : public ::testing::Test { webrtc::ScalabilityMode::kL3T3h, webrtc::ScalabilityMode::kL3T3_KEY, webrtc::ScalabilityMode::kS2T1, + webrtc::ScalabilityMode::kS2T1h, webrtc::ScalabilityMode::kS2T2, + webrtc::ScalabilityMode::kS2T2h, webrtc::ScalabilityMode::kS2T3, + webrtc::ScalabilityMode::kS2T3h, webrtc::ScalabilityMode::kS3T1, + webrtc::ScalabilityMode::kS3T1h, webrtc::ScalabilityMode::kS3T2, - webrtc::ScalabilityMode::kS3T3) + webrtc::ScalabilityMode::kS3T2h, + webrtc::ScalabilityMode::kS3T3, + webrtc::ScalabilityMode::kS3T3h) // clang-format on ) << "Codec: " << codec.name; diff --git a/video/video_send_stream_tests.cc b/video/video_send_stream_tests.cc index c948012c54..f4ba3df796 100644 --- a/video/video_send_stream_tests.cc +++ b/video/video_send_stream_tests.cc @@ -3392,7 +3392,13 @@ INSTANTIATE_TEST_SUITE_P( {"L2T2_KEY_SHIFT", 2, 2, InterLayerPredMode::kOnKeyPic}, {"L3T1h", 3, 1, InterLayerPredMode::kOn}, {"L3T2h", 3, 2, InterLayerPredMode::kOn}, - {"L3T3h", 3, 3, InterLayerPredMode::kOn}}), + {"L3T3h", 3, 3, InterLayerPredMode::kOn}, + {"S2T1h", 2, 1, InterLayerPredMode::kOff}, + {"S2T2h", 2, 2, InterLayerPredMode::kOff}, + {"S2T3h", 2, 3, InterLayerPredMode::kOff}, + {"S3T1h", 3, 1, InterLayerPredMode::kOff}, + {"S3T2h", 3, 2, InterLayerPredMode::kOff}, + {"S3T3h", 3, 3, InterLayerPredMode::kOff}}), ::testing::Values(true)), // use_scalability_mode_identifier ParamInfoToStr);