Add support for more scalability modes (1.5:1 resolution ratio).

Added modes:
- S2T1h
- S2T2h
- S2T3h
- S3T1h
- S3T2h
- S3T3h

Bug: webrtc:13960
Change-Id: I618a30c68b0ce1609847ee33a2298fe8fa0720c9
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/273664
Reviewed-by: Florent Castelli <orphis@webrtc.org>
Reviewed-by: Rasmus Brandt <brandtr@webrtc.org>
Commit-Queue: Åsa Persson <asapersson@webrtc.org>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#37968}

api/video_codecs/scalability_mode.cc

@@ -62,16 +62,28 @@ absl::string_view ScalabilityModeToString(ScalabilityMode scalability_mode) {
       return "L3T3_KEY";
     case ScalabilityMode::kS2T1:
       return "S2T1";
+    case ScalabilityMode::kS2T1h:
+      return "S2T1h";
     case ScalabilityMode::kS2T2:
       return "S2T2";
+    case ScalabilityMode::kS2T2h:
+      return "S2T2h";
     case ScalabilityMode::kS2T3:
       return "S2T3";
+    case ScalabilityMode::kS2T3h:
+      return "S2T3h";
     case ScalabilityMode::kS3T1:
       return "S3T1";
+    case ScalabilityMode::kS3T1h:
+      return "S3T1h";
     case ScalabilityMode::kS3T2:
       return "S3T2";
+    case ScalabilityMode::kS3T2h:
+      return "S3T2h";
     case ScalabilityMode::kS3T3:
       return "S3T3";
+    case ScalabilityMode::kS3T3h:
+      return "S3T3h";
   }
   RTC_CHECK_NOTREACHED();
 }

api/video_codecs/scalability_mode.h

@@ -48,11 +48,17 @@ enum class ScalabilityMode : uint8_t {
   kL3T3h,
   kL3T3_KEY,
   kS2T1,
+  kS2T1h,
   kS2T2,
+  kS2T2h,
   kS2T3,
+  kS2T3h,
   kS3T1,
+  kS3T1h,
   kS3T2,
+  kS3T2h,
   kS3T3,
+  kS3T3h,
 };
 
 inline constexpr ScalabilityMode kAllScalabilityModes[] = {
@@ -80,11 +86,17 @@ inline constexpr ScalabilityMode kAllScalabilityModes[] = {
     ScalabilityMode::kL3T3h,
     ScalabilityMode::kL3T3_KEY,
     ScalabilityMode::kS2T1,
+    ScalabilityMode::kS2T1h,
     ScalabilityMode::kS2T2,
+    ScalabilityMode::kS2T2h,
     ScalabilityMode::kS2T3,
+    ScalabilityMode::kS2T3h,
     ScalabilityMode::kS3T1,
+    ScalabilityMode::kS3T1h,
     ScalabilityMode::kS3T2,
+    ScalabilityMode::kS3T2h,
     ScalabilityMode::kS3T3,
+    ScalabilityMode::kS3T3h,
     // clang-format on
 };
 

modules/video_coding/svc/create_scalability_structure.cc

@@ -148,6 +148,13 @@ constexpr ScalableVideoController::StreamLayersConfig kConfigS2T1 = {
     {1, 1},
     {2, 1}};
 
+constexpr ScalableVideoController::StreamLayersConfig kConfigS2T1h = {
+    /*num_spatial_layers=*/2,
+    /*num_temporal_layers=*/1,
+    /*uses_reference_scaling=*/false,
+    {2, 1},
+    {3, 1}};
+
 constexpr ScalableVideoController::StreamLayersConfig kConfigS2T2 = {
     /*num_spatial_layers=*/2,
     /*num_temporal_layers=*/2,
@@ -155,6 +162,13 @@ constexpr ScalableVideoController::StreamLayersConfig kConfigS2T2 = {
     {1, 1},
     {2, 1}};
 
+constexpr ScalableVideoController::StreamLayersConfig kConfigS2T2h = {
+    /*num_spatial_layers=*/2,
+    /*num_temporal_layers=*/2,
+    /*uses_reference_scaling=*/false,
+    {2, 1},
+    {3, 1}};
+
 constexpr ScalableVideoController::StreamLayersConfig kConfigS2T3 = {
     /*num_spatial_layers=*/2,
     /*num_temporal_layers=*/3,
@@ -162,6 +176,13 @@ constexpr ScalableVideoController::StreamLayersConfig kConfigS2T3 = {
     {1, 1},
     {2, 1}};
 
+constexpr ScalableVideoController::StreamLayersConfig kConfigS2T3h = {
+    /*num_spatial_layers=*/2,
+    /*num_temporal_layers=*/3,
+    /*uses_reference_scaling=*/false,
+    {2, 1},
+    {3, 1}};
+
 constexpr ScalableVideoController::StreamLayersConfig kConfigS3T1 = {
     /*num_spatial_layers=*/3,
     /*num_temporal_layers=*/1,
@@ -169,6 +190,13 @@ constexpr ScalableVideoController::StreamLayersConfig kConfigS3T1 = {
     {1, 1, 1},
     {4, 2, 1}};
 
+constexpr ScalableVideoController::StreamLayersConfig kConfigS3T1h = {
+    /*num_spatial_layers=*/3,
+    /*num_temporal_layers=*/1,
+    /*uses_reference_scaling=*/false,
+    {4, 2, 1},
+    {9, 3, 1}};
+
 constexpr ScalableVideoController::StreamLayersConfig kConfigS3T2 = {
     /*num_spatial_layers=*/3,
     /*num_temporal_layers=*/2,
@@ -176,6 +204,13 @@ constexpr ScalableVideoController::StreamLayersConfig kConfigS3T2 = {
     {1, 1, 1},
     {4, 2, 1}};
 
+constexpr ScalableVideoController::StreamLayersConfig kConfigS3T2h = {
+    /*num_spatial_layers=*/3,
+    /*num_temporal_layers=*/2,
+    /*uses_reference_scaling=*/false,
+    {4, 2, 1},
+    {9, 3, 1}};
+
 constexpr ScalableVideoController::StreamLayersConfig kConfigS3T3 = {
     /*num_spatial_layers=*/3,
     /*num_temporal_layers=*/3,
@@ -183,6 +218,13 @@ constexpr ScalableVideoController::StreamLayersConfig kConfigS3T3 = {
     {1, 1, 1},
     {4, 2, 1}};
 
+constexpr ScalableVideoController::StreamLayersConfig kConfigS3T3h = {
+    /*num_spatial_layers=*/3,
+    /*num_temporal_layers=*/3,
+    /*uses_reference_scaling=*/false,
+    {4, 2, 1},
+    {9, 3, 1}};
+
 constexpr NamedStructureFactory kFactories[] = {
     {ScalabilityMode::kL1T1, Create<ScalableVideoControllerNoLayering>,
      kConfigL1T1},
@@ -215,11 +257,17 @@ constexpr NamedStructureFactory kFactories[] = {
     {ScalabilityMode::kL3T3_KEY, Create<ScalabilityStructureL3T3Key>,
      kConfigL3T3},
     {ScalabilityMode::kS2T1, Create<ScalabilityStructureS2T1>, kConfigS2T1},
+    {ScalabilityMode::kS2T1h, CreateH<ScalabilityStructureS2T1>, kConfigS2T1h},
     {ScalabilityMode::kS2T2, Create<ScalabilityStructureS2T2>, kConfigS2T2},
+    {ScalabilityMode::kS2T2h, CreateH<ScalabilityStructureS2T2>, kConfigS2T2h},
     {ScalabilityMode::kS2T3, Create<ScalabilityStructureS2T3>, kConfigS2T3},
+    {ScalabilityMode::kS2T3h, CreateH<ScalabilityStructureS2T3>, kConfigS2T3h},
     {ScalabilityMode::kS3T1, Create<ScalabilityStructureS3T1>, kConfigS3T1},
+    {ScalabilityMode::kS3T1h, CreateH<ScalabilityStructureS3T1>, kConfigS3T1h},
     {ScalabilityMode::kS3T2, Create<ScalabilityStructureS3T2>, kConfigS3T2},
+    {ScalabilityMode::kS3T2h, CreateH<ScalabilityStructureS3T2>, kConfigS3T2h},
     {ScalabilityMode::kS3T3, Create<ScalabilityStructureS3T3>, kConfigS3T3},
+    {ScalabilityMode::kS3T3h, CreateH<ScalabilityStructureS3T3>, kConfigS3T3h},
 };
 
 }  // namespace

modules/video_coding/svc/scalability_mode_util.cc

@@ -71,16 +71,28 @@ absl::optional<ScalabilityMode> ScalabilityModeFromString(
 
   if (mode_string == "S2T1")
     return ScalabilityMode::kS2T1;
+  if (mode_string == "S2T1h")
+    return ScalabilityMode::kS2T1h;
   if (mode_string == "S2T2")
     return ScalabilityMode::kS2T2;
+  if (mode_string == "S2T2h")
+    return ScalabilityMode::kS2T2h;
   if (mode_string == "S2T3")
     return ScalabilityMode::kS2T3;
+  if (mode_string == "S2T3h")
+    return ScalabilityMode::kS2T3h;
   if (mode_string == "S3T1")
     return ScalabilityMode::kS3T1;
+  if (mode_string == "S3T1h")
+    return ScalabilityMode::kS3T1h;
   if (mode_string == "S3T2")
     return ScalabilityMode::kS3T2;
+  if (mode_string == "S3T2h")
+    return ScalabilityMode::kS3T2h;
   if (mode_string == "S3T3")
     return ScalabilityMode::kS3T3;
+  if (mode_string == "S3T3h")
+    return ScalabilityMode::kS3T3h;
 
   return absl::nullopt;
 }
@@ -123,11 +135,17 @@ InterLayerPredMode ScalabilityModeToInterLayerPredMode(
     case ScalabilityMode::kL3T3_KEY:
       return InterLayerPredMode::kOnKeyPic;
     case ScalabilityMode::kS2T1:
+    case ScalabilityMode::kS2T1h:
     case ScalabilityMode::kS2T2:
+    case ScalabilityMode::kS2T2h:
     case ScalabilityMode::kS2T3:
+    case ScalabilityMode::kS2T3h:
     case ScalabilityMode::kS3T1:
+    case ScalabilityMode::kS3T1h:
     case ScalabilityMode::kS3T2:
+    case ScalabilityMode::kS3T2h:
     case ScalabilityMode::kS3T3:
+    case ScalabilityMode::kS3T3h:
       return InterLayerPredMode::kOff;
   }
   RTC_CHECK_NOTREACHED();
@@ -161,12 +179,18 @@ int ScalabilityModeToNumSpatialLayers(ScalabilityMode scalability_mode) {
     case ScalabilityMode::kL3T3_KEY:
       return 3;
     case ScalabilityMode::kS2T1:
+    case ScalabilityMode::kS2T1h:
     case ScalabilityMode::kS2T2:
+    case ScalabilityMode::kS2T2h:
     case ScalabilityMode::kS2T3:
+    case ScalabilityMode::kS2T3h:
       return 2;
     case ScalabilityMode::kS3T1:
+    case ScalabilityMode::kS3T1h:
     case ScalabilityMode::kS3T2:
+    case ScalabilityMode::kS3T2h:
     case ScalabilityMode::kS3T3:
+    case ScalabilityMode::kS3T3h:
       return 3;
   }
   RTC_CHECK_NOTREACHED();
@@ -206,13 +230,19 @@ int ScalabilityModeToNumTemporalLayers(ScalabilityMode scalability_mode) {
     case ScalabilityMode::kL3T3_KEY:
       return 3;
     case ScalabilityMode::kS2T1:
+    case ScalabilityMode::kS2T1h:
     case ScalabilityMode::kS3T1:
+    case ScalabilityMode::kS3T1h:
       return 1;
     case ScalabilityMode::kS2T2:
+    case ScalabilityMode::kS2T2h:
     case ScalabilityMode::kS3T2:
+    case ScalabilityMode::kS3T2h:
       return 2;
     case ScalabilityMode::kS2T3:
+    case ScalabilityMode::kS2T3h:
     case ScalabilityMode::kS3T3:
+    case ScalabilityMode::kS3T3h:
       return 3;
   }
   RTC_CHECK_NOTREACHED();
@@ -251,6 +281,12 @@ absl::optional<ScalabilityModeResolutionRatio> ScalabilityModeToResolutionRatio(
     case ScalabilityMode::kL3T1h:
     case ScalabilityMode::kL3T2h:
     case ScalabilityMode::kL3T3h:
+    case ScalabilityMode::kS2T1h:
+    case ScalabilityMode::kS2T2h:
+    case ScalabilityMode::kS2T3h:
+    case ScalabilityMode::kS3T1h:
+    case ScalabilityMode::kS3T2h:
+    case ScalabilityMode::kS3T3h:
       return ScalabilityModeResolutionRatio::kThreeToTwo;
   }
   RTC_CHECK_NOTREACHED();

modules/video_coding/svc/scalability_structure_simulcast.cc

@@ -43,9 +43,11 @@ constexpr int ScalabilityStructureSimulcast::kMaxNumTemporalLayers;
 
 ScalabilityStructureSimulcast::ScalabilityStructureSimulcast(
     int num_spatial_layers,
-    int num_temporal_layers)
+    int num_temporal_layers,
+    ScalingFactor resolution_factor)
     : num_spatial_layers_(num_spatial_layers),
       num_temporal_layers_(num_temporal_layers),
+      resolution_factor_(resolution_factor),
       active_decode_targets_(
           (uint32_t{1} << (num_spatial_layers * num_temporal_layers)) - 1) {
   RTC_DCHECK_LE(num_spatial_layers, kMaxNumSpatialLayers);
@@ -62,8 +64,10 @@ ScalabilityStructureSimulcast::StreamConfig() const {
   result.scaling_factor_num[num_spatial_layers_ - 1] = 1;
   result.scaling_factor_den[num_spatial_layers_ - 1] = 1;
   for (int sid = num_spatial_layers_ - 1; sid > 0; --sid) {
-    result.scaling_factor_num[sid - 1] = 1;
+    result.scaling_factor_num[sid - 1] =
-    result.scaling_factor_den[sid - 1] = 2 * result.scaling_factor_den[sid];
+        resolution_factor_.num * result.scaling_factor_num[sid];
+    result.scaling_factor_den[sid - 1] =
+        resolution_factor_.den * result.scaling_factor_den[sid];
   }
   result.uses_reference_scaling = false;
   return result;

modules/video_coding/svc/scalability_structure_simulcast.h

@@ -23,8 +23,13 @@ namespace webrtc {
 // same temporal layering.
 class ScalabilityStructureSimulcast : public ScalableVideoController {
  public:
+  struct ScalingFactor {
+    int num = 1;
+    int den = 2;
+  };
   ScalabilityStructureSimulcast(int num_spatial_layers,
-                                int num_temporal_layers);
+                                int num_temporal_layers,
+                                ScalingFactor resolution_factor);
   ~ScalabilityStructureSimulcast() override;
 
   StreamLayersConfig StreamConfig() const override;
@@ -58,6 +63,7 @@ class ScalabilityStructureSimulcast : public ScalableVideoController {
 
   const int num_spatial_layers_;
   const int num_temporal_layers_;
+  const ScalingFactor resolution_factor_;
 
   FramePattern last_pattern_ = kNone;
   std::bitset<kMaxNumSpatialLayers> can_reference_t0_frame_for_spatial_id_ = 0;
@@ -70,7 +76,8 @@ class ScalabilityStructureSimulcast : public ScalableVideoController {
 // S0  0--0--0-
 class ScalabilityStructureS2T1 : public ScalabilityStructureSimulcast {
  public:
-  ScalabilityStructureS2T1() : ScalabilityStructureSimulcast(2, 1) {}
+  explicit ScalabilityStructureS2T1(ScalingFactor resolution_factor = {})
+      : ScalabilityStructureSimulcast(2, 1, resolution_factor) {}
   ~ScalabilityStructureS2T1() override = default;
 
   FrameDependencyStructure DependencyStructure() const override;
@@ -78,7 +85,8 @@ class ScalabilityStructureS2T1 : public ScalabilityStructureSimulcast {
 
 class ScalabilityStructureS2T2 : public ScalabilityStructureSimulcast {
  public:
-  ScalabilityStructureS2T2() : ScalabilityStructureSimulcast(2, 2) {}
+  explicit ScalabilityStructureS2T2(ScalingFactor resolution_factor = {})
+      : ScalabilityStructureSimulcast(2, 2, resolution_factor) {}
   ~ScalabilityStructureS2T2() override = default;
 
   FrameDependencyStructure DependencyStructure() const override;
@@ -98,7 +106,8 @@ class ScalabilityStructureS2T2 : public ScalabilityStructureSimulcast {
 // Time->   0 1 2 3 4
 class ScalabilityStructureS2T3 : public ScalabilityStructureSimulcast {
  public:
-  ScalabilityStructureS2T3() : ScalabilityStructureSimulcast(2, 3) {}
+  explicit ScalabilityStructureS2T3(ScalingFactor resolution_factor = {})
+      : ScalabilityStructureSimulcast(2, 3, resolution_factor) {}
   ~ScalabilityStructureS2T3() override = default;
 
   FrameDependencyStructure DependencyStructure() const override;
@@ -106,7 +115,8 @@ class ScalabilityStructureS2T3 : public ScalabilityStructureSimulcast {
 
 class ScalabilityStructureS3T1 : public ScalabilityStructureSimulcast {
  public:
-  ScalabilityStructureS3T1() : ScalabilityStructureSimulcast(3, 1) {}
+  explicit ScalabilityStructureS3T1(ScalingFactor resolution_factor = {})
+      : ScalabilityStructureSimulcast(3, 1, resolution_factor) {}
   ~ScalabilityStructureS3T1() override = default;
 
   FrameDependencyStructure DependencyStructure() const override;
@@ -114,7 +124,8 @@ class ScalabilityStructureS3T1 : public ScalabilityStructureSimulcast {
 
 class ScalabilityStructureS3T2 : public ScalabilityStructureSimulcast {
  public:
-  ScalabilityStructureS3T2() : ScalabilityStructureSimulcast(3, 2) {}
+  explicit ScalabilityStructureS3T2(ScalingFactor resolution_factor = {})
+      : ScalabilityStructureSimulcast(3, 2, resolution_factor) {}
   ~ScalabilityStructureS3T2() override = default;
 
   FrameDependencyStructure DependencyStructure() const override;
@@ -122,7 +133,8 @@ class ScalabilityStructureS3T2 : public ScalabilityStructureSimulcast {
 
 class ScalabilityStructureS3T3 : public ScalabilityStructureSimulcast {
  public:
-  ScalabilityStructureS3T3() : ScalabilityStructureSimulcast(3, 3) {}
+  explicit ScalabilityStructureS3T3(ScalingFactor resolution_factor = {})
+      : ScalabilityStructureSimulcast(3, 3, resolution_factor) {}
   ~ScalabilityStructureS3T3() override = default;
 
   FrameDependencyStructure DependencyStructure() const override;

pc/peer_connection_factory_unittest.cc

@@ -222,11 +222,17 @@ class PeerConnectionFactoryTest : public ::testing::Test {
                 webrtc::ScalabilityMode::kL3T3h,
                 webrtc::ScalabilityMode::kL3T3_KEY,
                 webrtc::ScalabilityMode::kS2T1,
+                webrtc::ScalabilityMode::kS2T1h,
                 webrtc::ScalabilityMode::kS2T2,
+                webrtc::ScalabilityMode::kS2T2h,
                 webrtc::ScalabilityMode::kS2T3,
+                webrtc::ScalabilityMode::kS2T3h,
                 webrtc::ScalabilityMode::kS3T1,
+                webrtc::ScalabilityMode::kS3T1h,
                 webrtc::ScalabilityMode::kS3T2,
-                webrtc::ScalabilityMode::kS3T3)
+                webrtc::ScalabilityMode::kS3T2h,
+                webrtc::ScalabilityMode::kS3T3,
+                webrtc::ScalabilityMode::kS3T3h)
             // clang-format on
             )
             << "Codec: " << codec.name;

video/video_send_stream_tests.cc

@@ -3392,7 +3392,13 @@ INSTANTIATE_TEST_SUITE_P(
              {"L2T2_KEY_SHIFT", 2, 2, InterLayerPredMode::kOnKeyPic},
              {"L3T1h", 3, 1, InterLayerPredMode::kOn},
              {"L3T2h", 3, 2, InterLayerPredMode::kOn},
-             {"L3T3h", 3, 3, InterLayerPredMode::kOn}}),
+             {"L3T3h", 3, 3, InterLayerPredMode::kOn},
+             {"S2T1h", 2, 1, InterLayerPredMode::kOff},
+             {"S2T2h", 2, 2, InterLayerPredMode::kOff},
+             {"S2T3h", 2, 3, InterLayerPredMode::kOff},
+             {"S3T1h", 3, 1, InterLayerPredMode::kOff},
+             {"S3T2h", 3, 2, InterLayerPredMode::kOff},
+             {"S3T3h", 3, 3, InterLayerPredMode::kOff}}),
         ::testing::Values(true)),  // use_scalability_mode_identifier
     ParamInfoToStr);