mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-12 21:30:45 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

[Adaptation] Move deg.pref. out of ResourceAdaptationProcessor

Browse source 
This patch creates a new class which provides the DegradationPreference
thread safe to all classes that need if (BalancedConstraint and
QpScalerResource). It also broadcasts to all listeners when the
preferences are updated, so the ResourceAdaptationProcessor can update
the video if needed.

In future work, this could be used to remove the need for two task
queues for the VideoStreamEncoder resources.

Bug: webrtc:11700
Change-Id: I05480db8b7157b5643f6f86ec9c64850839b3e76
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/177522
Commit-Queue: Evan Shrubsole <eshr@google.com>
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#31623}
This commit is contained in:
Evan Shrubsole 2020-07-03 13:51:48 +02:00 committed by Commit Bot
parent db1c81d45b
commit 9492d500dd
17 changed files with 281 additions and 160 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
call/adaptation/BUILD.gn
View file

@ -16,6 +16,10 @@ rtc_library("resource_adaptation") {
"adaptation_listener.h",
"broadcast_resource_listener.cc",
"broadcast_resource_listener.h",
"degradation_preference_listener.cc",
"degradation_preference_listener.h",
"degradation_preference_provider.cc",
"degradation_preference_provider.h",
"encoder_settings.cc",
"encoder_settings.h",
"resource_adaptation_processor.cc",

17
call/adaptation/degradation_preference_listener.cc Normal file
View file

@ -0,0 +1,17 @@
/*
* Copyright 2020 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "call/adaptation/degradation_preference_listener.h"
namespace webrtc {
DegradationPreferenceListener::~DegradationPreferenceListener() = default;
} // namespace webrtc

28
call/adaptation/degradation_preference_listener.h Normal file
View file

@ -0,0 +1,28 @@
/*
* Copyright 2020 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef CALL_ADAPTATION_DEGRADATION_PREFERENCE_LISTENER_H_
#define CALL_ADAPTATION_DEGRADATION_PREFERENCE_LISTENER_H_
#include "api/rtp_parameters.h"
namespace webrtc {
class DegradationPreferenceListener {
public:
virtual ~DegradationPreferenceListener();
virtual void OnDegradationPreferenceUpdated(
DegradationPreference degradation_preference) = 0;
};
} // namespace webrtc
#endif // CALL_ADAPTATION_DEGRADATION_PREFERENCE_LISTENER_H_

14
call/adaptation/degradation_preference_provider.cc Normal file
View file

@ -0,0 +1,14 @@
/*
* Copyright 2020 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "call/adaptation/degradation_preference_provider.h"
webrtc::DegradationPreferenceProvider::~DegradationPreferenceProvider() =
default;

28
call/adaptation/degradation_preference_provider.h Normal file
View file

@ -0,0 +1,28 @@
/*
* Copyright 2020 The WebRTC Project Authors. All rights reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef CALL_ADAPTATION_DEGRADATION_PREFERENCE_PROVIDER_H_
#define CALL_ADAPTATION_DEGRADATION_PREFERENCE_PROVIDER_H_
#include "api/rtp_parameters.h"
namespace webrtc {
// Thread-safe retrieval of degradation preferences.
class DegradationPreferenceProvider {
public:
virtual ~DegradationPreferenceProvider();
virtual DegradationPreference degradation_preference() const = 0;
};
} // namespace webrtc
#endif // CALL_ADAPTATION_DEGRADATION_PREFERENCE_PROVIDER_H_

46
call/adaptation/resource_adaptation_processor.cc
View file

@ -75,9 +75,7 @@ ResourceAdaptationProcessor::ResourceAdaptationProcessor(
new rtc::RefCountedObject<ResourceListenerDelegate>(this)),
encoder_stats_observer_(encoder_stats_observer),
resources_(),
degradation_preference_(DegradationPreference::DISABLED),
effective_degradation_preference_(DegradationPreference::DISABLED),
is_screenshare_(false),
stream_adapter_(stream_adapter),
last_reported_source_restrictions_(),
previous_mitigation_results_(),
@ -112,18 +110,6 @@ void ResourceAdaptationProcessor::SetResourceAdaptationQueue(
stream_adapter_->AddRestrictionsListener(this);
}
DegradationPreference ResourceAdaptationProcessor::degradation_preference()
const {
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
return degradation_preference_;
}
DegradationPreference
ResourceAdaptationProcessor::effective_degradation_preference() const {
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
return effective_degradation_preference_;
}
void ResourceAdaptationProcessor::AddResourceLimitationsListener(
ResourceLimitationsListener* limitations_listener) {
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
@ -215,26 +201,20 @@ void ResourceAdaptationProcessor::RemoveAdaptationListener(
adaptation_listeners_.erase(it);
}
void ResourceAdaptationProcessor::SetDegradationPreference(
void ResourceAdaptationProcessor::OnDegradationPreferenceUpdated(
DegradationPreference degradation_preference) {
if (!resource_adaptation_queue_->IsCurrent()) {
resource_adaptation_queue_->PostTask(
ToQueuedTask([this, degradation_preference]() {
OnDegradationPreferenceUpdated(degradation_preference);
}));
return;
}
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
degradation_preference_ = degradation_preference;
MaybeUpdateEffectiveDegradationPreference();
}
void ResourceAdaptationProcessor::SetIsScreenshare(bool is_screenshare) {
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
is_screenshare_ = is_screenshare;
MaybeUpdateEffectiveDegradationPreference();
}
void ResourceAdaptationProcessor::MaybeUpdateEffectiveDegradationPreference() {
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
effective_degradation_preference_ =
(is_screenshare_ &&
degradation_preference_ == DegradationPreference::BALANCED)
? DegradationPreference::MAINTAIN_RESOLUTION
: degradation_preference_;
if (degradation_preference == effective_degradation_preference_) {
return;
}
effective_degradation_preference_ = degradation_preference;
stream_adapter_->SetDegradationPreference(effective_degradation_preference_);
}
@ -419,7 +399,7 @@ void ResourceAdaptationProcessor::TriggerAdaptationDueToFrameDroppedDueToSize(
VideoAdaptationCounters counters_before =
stream_adapter_->adaptation_counters();
OnResourceOveruse(reason_resource);
if (degradation_preference_ == DegradationPreference::BALANCED &&
if (effective_degradation_preference_ == DegradationPreference::BALANCED &&
stream_adapter_->adaptation_counters().fps_adaptations >
counters_before.fps_adaptations) {
// Oops, we adapted frame rate. Adapt again, maybe it will adapt resolution!

17
call/adaptation/resource_adaptation_processor.h
View file

@ -26,6 +26,7 @@
#include "api/video/video_stream_encoder_observer.h"
#include "call/adaptation/adaptation_constraint.h"
#include "call/adaptation/adaptation_listener.h"
#include "call/adaptation/degradation_preference_listener.h"
#include "call/adaptation/resource_adaptation_processor_interface.h"
#include "call/adaptation/video_source_restrictions.h"
#include "call/adaptation/video_stream_adapter.h"
@ -52,7 +53,8 @@ namespace webrtc {
// i.e. the "resource adaptation task queue".
class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface,
public VideoSourceRestrictionsListener,
public ResourceListener {
public ResourceListener,
public DegradationPreferenceListener {
public:
ResourceAdaptationProcessor(
VideoStreamEncoderObserver* encoder_stats_observer,
@ -63,9 +65,6 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface,
TaskQueueBase* resource_adaptation_queue) override;
// ResourceAdaptationProcessorInterface implementation.
DegradationPreference degradation_preference() const override;
DegradationPreference effective_degradation_preference() const override;
void AddResourceLimitationsListener(
ResourceLimitationsListener* limitations_listener) override;
void RemoveResourceLimitationsListener(
@ -81,10 +80,6 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface,
void RemoveAdaptationListener(
AdaptationListener* adaptation_listener) override;
void SetDegradationPreference(
DegradationPreference degradation_preference) override;
void SetIsScreenshare(bool is_screenshare) override;
// ResourceListener implementation.
// Triggers OnResourceUnderuse() or OnResourceOveruse().
void OnResourceUsageStateMeasured(rtc::scoped_refptr<Resource> resource,
@ -96,6 +91,9 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface,
const VideoAdaptationCounters& adaptation_counters,
rtc::scoped_refptr<Resource> reason,
const VideoSourceRestrictions& unfiltered_restrictions) override;
// DegradationPreferenceListener implementation.
void OnDegradationPreferenceUpdated(
DegradationPreference degradation_preference) override;
// May trigger 1-2 adaptations. It is meant to reduce resolution but this is
// not guaranteed. It may adapt frame rate, which does not address the issue.
@ -190,11 +188,8 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface,
adaptation_limits_by_resources_
RTC_GUARDED_BY(resource_adaptation_queue_);
// Adaptation strategy settings.
DegradationPreference degradation_preference_
RTC_GUARDED_BY(resource_adaptation_queue_);
DegradationPreference effective_degradation_preference_
RTC_GUARDED_BY(resource_adaptation_queue_);
bool is_screenshare_ RTC_GUARDED_BY(resource_adaptation_queue_);
// Responsible for generating and applying possible adaptations.
VideoStreamAdapter* const stream_adapter_
RTC_GUARDED_BY(resource_adaptation_queue_);

11
call/adaptation/resource_adaptation_processor_interface.h
View file

@ -51,13 +51,6 @@ class ResourceAdaptationProcessorInterface {
virtual void SetResourceAdaptationQueue(
TaskQueueBase* resource_adaptation_queue) = 0;
virtual DegradationPreference degradation_preference() const = 0;
// Reinterprets "balanced + screenshare" as "maintain-resolution".
// TODO(hbos): Don't do this. This is not what "balanced" means. If the
// application wants to maintain resolution it should set that degradation
// preference rather than depend on non-standard behaviors.
virtual DegradationPreference effective_degradation_preference() const = 0;
// Starts or stops listening to resources, effectively enabling or disabling
// processing.
// TODO(https://crbug.com/webrtc/11172): Automatically register and unregister
@ -80,10 +73,6 @@ class ResourceAdaptationProcessorInterface {
virtual void RemoveAdaptationListener(
AdaptationListener* adaptation_listener) = 0;
virtual void SetDegradationPreference(
DegradationPreference degradation_preference) = 0;
virtual void SetIsScreenshare(bool is_screenshare) = 0;
// May trigger one or more adaptations. It is meant to reduce resolution -
// useful if a frame was dropped due to its size - however, the implementation
// may not guarantee this (see resource_adaptation_processor.h).

66
call/adaptation/resource_adaptation_processor_unittest.cc
View file

@ -156,10 +156,6 @@ class ResourceAdaptationProcessorTest : public ::testing::Test {
} // namespace
TEST_F(ResourceAdaptationProcessorTest, DisabledByDefault) {
EXPECT_EQ(DegradationPreference::DISABLED,
processor_->degradation_preference());
EXPECT_EQ(DegradationPreference::DISABLED,
processor_->effective_degradation_preference());
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
// Adaptation does not happen when disabled.
resource_->SetUsageState(ResourceUsageState::kOveruse);
@ -167,7 +163,7 @@ TEST_F(ResourceAdaptationProcessorTest, DisabledByDefault) {
}
TEST_F(ResourceAdaptationProcessorTest, InsufficientInput) {
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
// Adaptation does not happen if input is insufficient.
// When frame size is missing (OnFrameSizeObserved not called yet).
@ -187,7 +183,7 @@ TEST_F(ResourceAdaptationProcessorTest, InsufficientInput) {
// restrictions. For that, see video_stream_adapter_unittest.cc.
TEST_F(ResourceAdaptationProcessorTest,
OveruseTriggersRestrictingResolutionInMaintainFrameRate) {
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
resource_->SetUsageState(ResourceUsageState::kOveruse);
@ -198,7 +194,7 @@ TEST_F(ResourceAdaptationProcessorTest,
TEST_F(ResourceAdaptationProcessorTest,
OveruseTriggersRestrictingFrameRateInMaintainResolution) {
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_RESOLUTION);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
resource_->SetUsageState(ResourceUsageState::kOveruse);
@ -209,7 +205,7 @@ TEST_F(ResourceAdaptationProcessorTest,
TEST_F(ResourceAdaptationProcessorTest,
OveruseTriggersRestrictingFrameRateAndResolutionInBalanced) {
processor_->SetDegradationPreference(DegradationPreference::BALANCED);
processor_->OnDegradationPreferenceUpdated(DegradationPreference::BALANCED);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
// Adapting multiple times eventually resticts both frame rate and
// resolution. Exactly many times we need to adapt depends on
@ -227,7 +223,7 @@ TEST_F(ResourceAdaptationProcessorTest,
}
TEST_F(ResourceAdaptationProcessorTest, AwaitingPreviousAdaptation) {
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
resource_->SetUsageState(ResourceUsageState::kOveruse);
@ -239,7 +235,7 @@ TEST_F(ResourceAdaptationProcessorTest, AwaitingPreviousAdaptation) {
}
TEST_F(ResourceAdaptationProcessorTest, CannotAdaptUpWhenUnrestricted) {
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
resource_->SetUsageState(ResourceUsageState::kUnderuse);
@ -247,7 +243,7 @@ TEST_F(ResourceAdaptationProcessorTest, CannotAdaptUpWhenUnrestricted) {
}
TEST_F(ResourceAdaptationProcessorTest, UnderuseTakesUsBackToUnrestricted) {
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
resource_->SetUsageState(ResourceUsageState::kOveruse);
@ -259,7 +255,7 @@ TEST_F(ResourceAdaptationProcessorTest, UnderuseTakesUsBackToUnrestricted) {
}
TEST_F(ResourceAdaptationProcessorTest, ResourcesCanPreventAdaptingUp) {
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
// Adapt down so that we can adapt up.
@ -274,7 +270,7 @@ TEST_F(ResourceAdaptationProcessorTest, ResourcesCanPreventAdaptingUp) {
TEST_F(ResourceAdaptationProcessorTest,
ResourcesCanNotAdaptUpIfNeverAdaptedDown) {
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
resource_->SetUsageState(ResourceUsageState::kOveruse);
@ -288,7 +284,7 @@ TEST_F(ResourceAdaptationProcessorTest,
TEST_F(ResourceAdaptationProcessorTest,
ResourcesCanNotAdaptUpIfNotAdaptedDownAfterReset) {
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
resource_->SetUsageState(ResourceUsageState::kOveruse);
@ -307,7 +303,7 @@ TEST_F(ResourceAdaptationProcessorTest,
}
TEST_F(ResourceAdaptationProcessorTest, OnlyMostLimitedResourceMayAdaptUp) {
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
resource_->SetUsageState(ResourceUsageState::kOveruse);
@ -337,7 +333,7 @@ TEST_F(ResourceAdaptationProcessorTest, OnlyMostLimitedResourceMayAdaptUp) {
TEST_F(ResourceAdaptationProcessorTest,
MultipleResourcesCanTriggerMultipleAdaptations) {
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
resource_->SetUsageState(ResourceUsageState::kOveruse);
@ -395,7 +391,7 @@ TEST_F(ResourceAdaptationProcessorTest,
TEST_F(ResourceAdaptationProcessorTest,
MostLimitedResourceAdaptationWorksAfterChangingDegradataionPreference) {
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
// Adapt down until we can't anymore.
@ -411,7 +407,7 @@ TEST_F(ResourceAdaptationProcessorTest,
RestrictSource(restrictions_listener_.restrictions());
int last_total = restrictions_listener_.adaptation_counters().Total();
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_RESOLUTION);
// resource_ can not adapt up since we have never reduced FPS.
resource_->SetUsageState(ResourceUsageState::kUnderuse);
@ -427,7 +423,7 @@ TEST_F(ResourceAdaptationProcessorTest,
}
TEST_F(ResourceAdaptationProcessorTest, AdaptingTriggersOnAdaptationApplied) {
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
resource_->SetUsageState(ResourceUsageState::kOveruse);
@ -436,7 +432,7 @@ TEST_F(ResourceAdaptationProcessorTest, AdaptingTriggersOnAdaptationApplied) {
TEST_F(ResourceAdaptationProcessorTest,
AdaptsDownWhenOtherResourceIsAlwaysUnderused) {
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
other_resource_->SetUsageState(ResourceUsageState::kUnderuse);
@ -457,7 +453,7 @@ TEST_F(ResourceAdaptationProcessorTest,
TEST_F(ResourceAdaptationProcessorTest,
TriggerOveruseNotOnAdaptationTaskQueue) {
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
@ -471,7 +467,7 @@ TEST_F(ResourceAdaptationProcessorTest,
TEST_F(ResourceAdaptationProcessorTest,
DestroyProcessorWhileResourceListenerDelegateHasTaskInFlight) {
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
@ -495,7 +491,7 @@ TEST_F(ResourceAdaptationProcessorTest,
TEST_F(ResourceAdaptationProcessorTest,
ResourceOveruseIgnoredWhenSignalledDuringRemoval) {
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
@ -524,7 +520,7 @@ TEST_F(ResourceAdaptationProcessorTest,
TEST_F(ResourceAdaptationProcessorTest,
RemovingOnlyAdaptedResourceResetsAdaptation) {
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
@ -541,7 +537,7 @@ TEST_F(ResourceAdaptationProcessorTest,
TEST_F(ResourceAdaptationProcessorTest,
RemovingMostLimitedResourceSetsAdaptationToNextLimitedLevel) {
processor_->SetDegradationPreference(DegradationPreference::BALANCED);
processor_->OnDegradationPreferenceUpdated(DegradationPreference::BALANCED);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
other_resource_->SetUsageState(ResourceUsageState::kOveruse);
@ -569,7 +565,7 @@ TEST_F(ResourceAdaptationProcessorTest,
TEST_F(ResourceAdaptationProcessorTest,
RemovingMostLimitedResourceSetsAdaptationIfInputStateUnchanged) {
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
@ -606,7 +602,7 @@ TEST_F(ResourceAdaptationProcessorTest,
TEST_F(ResourceAdaptationProcessorTest,
RemovingResourceNotMostLimitedHasNoEffectOnLimitations) {
processor_->SetDegradationPreference(DegradationPreference::BALANCED);
processor_->OnDegradationPreferenceUpdated(DegradationPreference::BALANCED);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
other_resource_->SetUsageState(ResourceUsageState::kOveruse);
@ -632,7 +628,7 @@ TEST_F(ResourceAdaptationProcessorTest,
TEST_F(ResourceAdaptationProcessorTest,
RemovingMostLimitedResourceAfterSwitchingDegradationPreferences) {
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
@ -644,7 +640,7 @@ TEST_F(ResourceAdaptationProcessorTest,
VideoAdaptationCounters next_limited_counters =
restrictions_listener_.adaptation_counters();
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_RESOLUTION);
resource_->SetUsageState(ResourceUsageState::kOveruse);
RestrictSource(restrictions_listener_.restrictions());
@ -658,7 +654,7 @@ TEST_F(ResourceAdaptationProcessorTest,
// After switching back to MAINTAIN_FRAMERATE, the next most limited settings
// are restored.
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
EXPECT_EQ(next_limited_restrictions, restrictions_listener_.restrictions());
@ -668,7 +664,7 @@ TEST_F(ResourceAdaptationProcessorTest,
TEST_F(ResourceAdaptationProcessorTest,
RemovingMostLimitedResourceSetsNextLimitationsInDisabled) {
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
@ -683,7 +679,7 @@ TEST_F(ResourceAdaptationProcessorTest,
RestrictSource(restrictions_listener_.restrictions());
EXPECT_EQ(2, restrictions_listener_.adaptation_counters().Total());
processor_->SetDegradationPreference(DegradationPreference::DISABLED);
processor_->OnDegradationPreferenceUpdated(DegradationPreference::DISABLED);
// Revert to |other_resource_| when removing |resource_| even though the
// current degradataion preference is disabled.
@ -691,7 +687,7 @@ TEST_F(ResourceAdaptationProcessorTest,
// After switching back to MAINTAIN_FRAMERATE, the next most limited settings
// are restored.
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
EXPECT_EQ(next_limited_restrictions, restrictions_listener_.restrictions());
EXPECT_EQ(next_limited_counters,
@ -703,7 +699,7 @@ TEST_F(ResourceAdaptationProcessorTest,
TEST_F(ResourceAdaptationProcessorTest,
RemovedResourceSignalsIgnoredByProcessor) {
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);
@ -717,7 +713,7 @@ TEST_F(ResourceAdaptationProcessorTest,
TEST_F(ResourceAdaptationProcessorTest,
RemovingResourceWhenMultipleMostLimtedHasNoEffect) {
processor_->SetDegradationPreference(
processor_->OnDegradationPreferenceUpdated(
DegradationPreference::MAINTAIN_FRAMERATE);
SetInputStates(true, kDefaultFrameRate, kDefaultFrameSize);

1
call/adaptation/video_stream_adapter.h
View file

@ -18,6 +18,7 @@
#include "api/adaptation/resource.h"
#include "api/rtp_parameters.h"
#include "api/video/video_adaptation_counters.h"
#include "call/adaptation/degradation_preference_provider.h"
#include "call/adaptation/video_source_restrictions.h"
#include "call/adaptation/video_stream_input_state.h"
#include "call/adaptation/video_stream_input_state_provider.h"

24
video/adaptation/quality_scaler_resource.cc
View file

@ -26,30 +26,29 @@ const int64_t kUnderuseDueToDisabledCooldownMs = 1000;
} // namespace
// static
rtc::scoped_refptr<QualityScalerResource> QualityScalerResource::Create() {
return new rtc::RefCountedObject<QualityScalerResource>();
rtc::scoped_refptr<QualityScalerResource> QualityScalerResource::Create(
DegradationPreferenceProvider* degradation_preference_provider) {
return new rtc::RefCountedObject<QualityScalerResource>(
degradation_preference_provider);
}
QualityScalerResource::QualityScalerResource()
QualityScalerResource::QualityScalerResource(
DegradationPreferenceProvider* degradation_preference_provider)
: VideoStreamEncoderResource("QualityScalerResource"),
quality_scaler_(nullptr),
last_underuse_due_to_disabled_timestamp_ms_(absl::nullopt),
num_handled_callbacks_(0),
pending_callbacks_(),
adaptation_processor_(nullptr),
clear_qp_samples_(false) {}
degradation_preference_provider_(degradation_preference_provider),
clear_qp_samples_(false) {
RTC_CHECK(degradation_preference_provider_);
}
QualityScalerResource::~QualityScalerResource() {
RTC_DCHECK(!quality_scaler_);
RTC_DCHECK(pending_callbacks_.empty());
}
void QualityScalerResource::SetAdaptationProcessor(
ResourceAdaptationProcessorInterface* adaptation_processor) {
RTC_DCHECK_RUN_ON(resource_adaptation_queue());
adaptation_processor_ = adaptation_processor;
}
bool QualityScalerResource::is_started() const {
RTC_DCHECK_RUN_ON(encoder_queue());
return quality_scaler_.get();
@ -183,8 +182,7 @@ void QualityScalerResource::OnAdaptationApplied(
// interval whose delay is calculated based on events such as these. Now there
// is much dependency on a specific OnReportQpUsageHigh() event and "balanced"
// but adaptations happening might not align with QualityScaler's CheckQpTask.
if (adaptation_processor_ &&
adaptation_processor_->effective_degradation_preference() ==
if (degradation_preference_provider_->degradation_preference() ==
DegradationPreference::BALANCED &&
DidDecreaseFrameRate(restrictions_before, restrictions_after)) {
absl::optional<int> min_diff = BalancedDegradationSettings().MinFpsDiff(

13
video/adaptation/quality_scaler_resource.h
View file

@ -20,6 +20,7 @@
#include "api/video/video_adaptation_reason.h"
#include "api/video_codecs/video_encoder.h"
#include "call/adaptation/adaptation_listener.h"
#include "call/adaptation/degradation_preference_provider.h"
#include "call/adaptation/resource_adaptation_processor_interface.h"
#include "modules/video_coding/utility/quality_scaler.h"
#include "rtc_base/critical_section.h"
@ -34,14 +35,13 @@ class QualityScalerResource : public VideoStreamEncoderResource,
public AdaptationListener,
public QualityScalerQpUsageHandlerInterface {
public:
static rtc::scoped_refptr<QualityScalerResource> Create();
static rtc::scoped_refptr<QualityScalerResource> Create(
DegradationPreferenceProvider* degradation_preference_provider);
QualityScalerResource();
explicit QualityScalerResource(
DegradationPreferenceProvider* degradation_preference_provider);
~QualityScalerResource() override;
void SetAdaptationProcessor(
ResourceAdaptationProcessorInterface* adaptation_processor);
bool is_started() const;
void StartCheckForOveruse(VideoEncoder::QpThresholds qp_thresholds);
@ -91,10 +91,9 @@ class QualityScalerResource : public VideoStreamEncoderResource,
size_t num_handled_callbacks_ RTC_GUARDED_BY(encoder_queue());
std::queue<rtc::scoped_refptr<QualityScalerQpUsageHandlerCallbackInterface>>
pending_callbacks_ RTC_GUARDED_BY(encoder_queue());
DegradationPreferenceProvider* const degradation_preference_provider_;
// Members accessed on the adaptation queue.
ResourceAdaptationProcessorInterface* adaptation_processor_
RTC_GUARDED_BY(resource_adaptation_queue());
bool clear_qp_samples_ RTC_GUARDED_BY(resource_adaptation_queue());
};

14
video/adaptation/quality_scaler_resource_unittest.cc
View file

@ -62,6 +62,15 @@ class FakeQualityScalerQpUsageHandlerCallback
absl::optional<bool> clear_qp_samples_result_;
};
class FakeDegradationPreferenceProvider : public DegradationPreferenceProvider {
public:
~FakeDegradationPreferenceProvider() override {}
DegradationPreference degradation_preference() const override {
return DegradationPreference::MAINTAIN_FRAMERATE;
}
};
} // namespace
class QualityScalerResourceTest : public ::testing::Test {
@ -74,7 +83,9 @@ class QualityScalerResourceTest : public ::testing::Test {
encoder_queue_(task_queue_factory_->CreateTaskQueue(
"EncoderQueue",
TaskQueueFactory::Priority::NORMAL)),
quality_scaler_resource_(QualityScalerResource::Create()) {
degradation_preference_provider_(),
quality_scaler_resource_(
QualityScalerResource::Create(&degradation_preference_provider_)) {
quality_scaler_resource_->RegisterEncoderTaskQueue(encoder_queue_.Get());
quality_scaler_resource_->RegisterAdaptationTaskQueue(
resource_adaptation_queue_.Get());
@ -100,6 +111,7 @@ class QualityScalerResourceTest : public ::testing::Test {
const std::unique_ptr<TaskQueueFactory> task_queue_factory_;
rtc::TaskQueue resource_adaptation_queue_;
rtc::TaskQueue encoder_queue_;
FakeDegradationPreferenceProvider degradation_preference_provider_;
rtc::scoped_refptr<QualityScalerResource> quality_scaler_resource_;
};

38
video/adaptation/video_stream_encoder_resource_manager.cc
View file

@ -198,24 +198,21 @@ bool VideoStreamEncoderResourceManager::BitrateConstraint::
}
VideoStreamEncoderResourceManager::BalancedConstraint::BalancedConstraint(
VideoStreamEncoderResourceManager* manager)
VideoStreamEncoderResourceManager* manager,
DegradationPreferenceProvider* degradation_preference_provider)
: manager_(manager),
resource_adaptation_queue_(nullptr),
adaptation_processor_(nullptr),
encoder_target_bitrate_bps_(absl::nullopt) {}
encoder_target_bitrate_bps_(absl::nullopt),
degradation_preference_provider_(degradation_preference_provider) {
RTC_DCHECK(manager_);
RTC_DCHECK(degradation_preference_provider_);
}
void VideoStreamEncoderResourceManager::BalancedConstraint::SetAdaptationQueue(
TaskQueueBase* resource_adaptation_queue) {
resource_adaptation_queue_ = resource_adaptation_queue;
}
void VideoStreamEncoderResourceManager::BalancedConstraint::
SetAdaptationProcessor(
ResourceAdaptationProcessorInterface* adaptation_processor) {
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
adaptation_processor_ = adaptation_processor;
}
void VideoStreamEncoderResourceManager::BalancedConstraint::
OnEncoderTargetBitrateUpdated(
absl::optional<uint32_t> encoder_target_bitrate_bps) {
@ -234,7 +231,6 @@ bool VideoStreamEncoderResourceManager::BalancedConstraint::
const VideoSourceRestrictions& restrictions_after,
rtc::scoped_refptr<Resource> reason_resource) const {
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
RTC_DCHECK(adaptation_processor_);
VideoAdaptationReason reason =
manager_->GetReasonFromResource(reason_resource);
// Don't adapt if BalancedDegradationSettings applies and determines this will
@ -242,7 +238,7 @@ bool VideoStreamEncoderResourceManager::BalancedConstraint::
// TODO(hbos): Why are we allowing violating balanced settings if adapting due
// CPU? Shouldn't this condition be checked regardless of reason?
if (reason == VideoAdaptationReason::kQuality &&
adaptation_processor_->effective_degradation_preference() ==
degradation_preference_provider_->degradation_preference() ==
DegradationPreference::BALANCED &&
!manager_->balanced_settings_.CanAdaptUp(
input_state.video_codec_type(),
@ -266,12 +262,17 @@ VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager(
VideoStreamEncoderObserver* encoder_stats_observer,
Clock* clock,
bool experiment_cpu_load_estimator,
std::unique_ptr<OveruseFrameDetector> overuse_detector)
: bitrate_constraint_(new rtc::RefCountedObject<BitrateConstraint>(this)),
balanced_constraint_(new rtc::RefCountedObject<BalancedConstraint>(this)),
std::unique_ptr<OveruseFrameDetector> overuse_detector,
DegradationPreferenceProvider* degradation_preference_provider)
: degradation_preference_provider_(degradation_preference_provider),
bitrate_constraint_(new rtc::RefCountedObject<BitrateConstraint>(this)),
balanced_constraint_(new rtc::RefCountedObject<BalancedConstraint>(
this,
degradation_preference_provider_)),
encode_usage_resource_(
EncodeUsageResource::Create(std::move(overuse_detector))),
quality_scaler_resource_(QualityScalerResource::Create()),
quality_scaler_resource_(
QualityScalerResource::Create(degradation_preference_provider_)),
encoder_queue_(nullptr),
resource_adaptation_queue_(nullptr),
input_state_provider_(input_state_provider),
@ -288,7 +289,8 @@ VideoStreamEncoderResourceManager::VideoStreamEncoderResourceManager(
quality_rampup_experiment_(
QualityRampUpExperimentHelper::CreateIfEnabled(this, clock_)),
encoder_settings_(absl::nullopt) {
RTC_DCHECK(encoder_stats_observer_);
RTC_CHECK(degradation_preference_provider_);
RTC_CHECK(encoder_stats_observer_);
MapResourceToReason(encode_usage_resource_, VideoAdaptationReason::kCpu);
MapResourceToReason(quality_scaler_resource_,
VideoAdaptationReason::kQuality);
@ -320,8 +322,6 @@ void VideoStreamEncoderResourceManager::SetAdaptationProcessor(
VideoStreamAdapter* stream_adapter) {
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
adaptation_processor_ = adaptation_processor;
balanced_constraint_->SetAdaptationProcessor(adaptation_processor);
quality_scaler_resource_->SetAdaptationProcessor(adaptation_processor);
stream_adapter_ = stream_adapter;
}

13
video/adaptation/video_stream_encoder_resource_manager.h
View file

@ -73,7 +73,8 @@ class VideoStreamEncoderResourceManager
VideoStreamEncoderObserver* encoder_stats_observer,
Clock* clock,
bool experiment_cpu_load_estimator,
std::unique_ptr<OveruseFrameDetector> overuse_detector);
std::unique_ptr<OveruseFrameDetector> overuse_detector,
DegradationPreferenceProvider* degradation_preference_provider);
~VideoStreamEncoderResourceManager() override;
void Initialize(rtc::TaskQueue* encoder_queue,
@ -202,12 +203,12 @@ class VideoStreamEncoderResourceManager
class BalancedConstraint : public rtc::RefCountInterface,
public AdaptationConstraint {
public:
explicit BalancedConstraint(VideoStreamEncoderResourceManager* manager);
BalancedConstraint(
VideoStreamEncoderResourceManager* manager,
DegradationPreferenceProvider* degradation_preference_provider);
~BalancedConstraint() override = default;
void SetAdaptationQueue(TaskQueueBase* resource_adaptation_queue);
void SetAdaptationProcessor(
ResourceAdaptationProcessorInterface* adaptation_processor);
void OnEncoderTargetBitrateUpdated(
absl::optional<uint32_t> encoder_target_bitrate_bps);
@ -224,12 +225,12 @@ class VideoStreamEncoderResourceManager
// ResourceAdaptationProcessor, i.e. when IsAdaptationUpAllowed() is called.
VideoStreamEncoderResourceManager* const manager_;
TaskQueueBase* resource_adaptation_queue_;
ResourceAdaptationProcessorInterface* adaptation_processor_
RTC_GUARDED_BY(resource_adaptation_queue_);
absl::optional<uint32_t> encoder_target_bitrate_bps_
RTC_GUARDED_BY(resource_adaptation_queue_);
DegradationPreferenceProvider* degradation_preference_provider_;
};
DegradationPreferenceProvider* const degradation_preference_provider_;
const rtc::scoped_refptr<BitrateConstraint> bitrate_constraint_;
const rtc::scoped_refptr<BalancedConstraint> balanced_constraint_;
const rtc::scoped_refptr<EncodeUsageResource> encode_usage_resource_;

98
video/video_stream_encoder.cc
View file

@ -201,6 +201,72 @@ bool VideoStreamEncoder::EncoderRateSettings::operator!=(
return !(*this == rhs);
}
class VideoStreamEncoder::DegradationPreferenceManager
: public DegradationPreferenceProvider {
public:
DegradationPreferenceManager()
: degradation_preference_(DegradationPreference::DISABLED),
is_screenshare_(false),
effective_degradation_preference_(DegradationPreference::DISABLED) {}
~DegradationPreferenceManager() override { RTC_DCHECK(listeners_.empty()); }
DegradationPreference degradation_preference() const override {
rtc::CritScope crit(&lock_);
return effective_degradation_preference_;
}
void SetDegradationPreference(DegradationPreference degradation_preference) {
rtc::CritScope crit(&lock_);
degradation_preference_ = degradation_preference;
MaybeUpdateEffectiveDegradationPreference();
}
void SetIsScreenshare(bool is_screenshare) {
rtc::CritScope crit(&lock_);
is_screenshare_ = is_screenshare;
MaybeUpdateEffectiveDegradationPreference();
}
void AddListener(DegradationPreferenceListener* listener) {
rtc::CritScope crit(&lock_);
RTC_DCHECK(absl::c_find(listeners_, listener) == listeners_.end());
listeners_.push_back(listener);
}
void RemoveListener(DegradationPreferenceListener* listener) {
rtc::CritScope crit(&lock_);
auto it = absl::c_find(listeners_, listener);
RTC_DCHECK(it != listeners_.end());
listeners_.erase(it);
}
private:
void MaybeUpdateEffectiveDegradationPreference()
RTC_EXCLUSIVE_LOCKS_REQUIRED(&lock_) {
DegradationPreference effective_degradation_preference =
(is_screenshare_ &&
degradation_preference_ == DegradationPreference::BALANCED)
? DegradationPreference::MAINTAIN_RESOLUTION
: degradation_preference_;
if (effective_degradation_preference != effective_degradation_preference_) {
effective_degradation_preference_ = effective_degradation_preference;
for (auto& listener : listeners_) {
listener->OnDegradationPreferenceUpdated(
effective_degradation_preference_);
}
}
}
rtc::CriticalSection lock_;
DegradationPreference degradation_preference_ RTC_GUARDED_BY(&lock_);
bool is_screenshare_ RTC_GUARDED_BY(&lock_);
DegradationPreference effective_degradation_preference_
RTC_GUARDED_BY(&lock_);
std::vector<DegradationPreferenceListener*> listeners_ RTC_GUARDED_BY(&lock_);
};
VideoStreamEncoder::VideoStreamEncoder(
Clock* clock,
uint32_t number_of_cores,
@ -262,13 +328,16 @@ VideoStreamEncoder::VideoStreamEncoder(
std::make_unique<ResourceAdaptationProcessor>(
encoder_stats_observer,
video_stream_adapter_.get())),
degradation_preference_manager_(
std::make_unique<DegradationPreferenceManager>()),
adaptation_constraints_(),
adaptation_listeners_(),
stream_resource_manager_(&input_state_provider_,
encoder_stats_observer,
clock_,
settings_.experiment_cpu_load_estimator,
std::move(overuse_detector)),
std::move(overuse_detector),
degradation_preference_manager_.get()),
video_source_sink_controller_(/*sink=*/this,
/*source=*/nullptr),
resource_adaptation_queue_(task_queue_factory->CreateTaskQueue(
@ -294,6 +363,8 @@ VideoStreamEncoder::VideoStreamEncoder(
&stream_resource_manager_);
video_stream_adapter_->AddRestrictionsListener(&stream_resource_manager_);
video_stream_adapter_->AddRestrictionsListener(this);
degradation_preference_manager_->AddListener(
resource_adaptation_processor_.get());
// Add the stream resource manager's resources to the processor.
adaptation_constraints_ = stream_resource_manager_.AdaptationConstraints();
@ -342,6 +413,8 @@ void VideoStreamEncoder::Stop() {
resource_adaptation_processor_->RemoveResourceLimitationsListener(
&stream_resource_manager_);
stream_resource_manager_.SetAdaptationProcessor(nullptr, nullptr);
degradation_preference_manager_->RemoveListener(
resource_adaptation_processor_.get());
resource_adaptation_processor_.reset();
}
shutdown_adaptation_processor_event.Set();
@ -434,17 +507,8 @@ void VideoStreamEncoder::SetSource(
video_source_sink_controller_.SetSource(source);
input_state_provider_.OnHasInputChanged(source);
// Set the degradation preference on the adaptation queue.
resource_adaptation_queue_.PostTask([this, degradation_preference] {
RTC_DCHECK_RUN_ON(&resource_adaptation_queue_);
if (!resource_adaptation_processor_) {
// The VideoStreamEncoder was stopped and the processor destroyed before
// this task had a chance to execute. No action needed.
return;
}
resource_adaptation_processor_->SetDegradationPreference(
degradation_preference);
});
degradation_preference_manager_->SetDegradationPreference(
degradation_preference);
// This may trigger reconfiguring the QualityScaler on the encoder queue.
encoder_queue_.PostTask([this, degradation_preference] {
RTC_DCHECK_RUN_ON(&encoder_queue_);
@ -870,15 +934,7 @@ void VideoStreamEncoder::OnEncoderSettingsChanged() {
input_state_provider_.OnEncoderSettingsChanged(encoder_settings);
bool is_screenshare = encoder_settings.encoder_config().content_type ==
VideoEncoderConfig::ContentType::kScreen;
resource_adaptation_queue_.PostTask([this, is_screenshare] {
RTC_DCHECK_RUN_ON(&resource_adaptation_queue_);
if (!resource_adaptation_processor_) {
// The VideoStreamEncoder was stopped and the processor destroyed before
// this task had a chance to execute. No action needed.
return;
}
resource_adaptation_processor_->SetIsScreenshare(is_screenshare);
});
degradation_preference_manager_->SetIsScreenshare(is_screenshare);
}
void VideoStreamEncoder::OnFrame(const VideoFrame& video_frame) {

9
video/video_stream_encoder.h
View file

@ -29,6 +29,7 @@
#include "api/video_codecs/video_encoder.h"
#include "call/adaptation/adaptation_constraint.h"
#include "call/adaptation/adaptation_listener.h"
#include "call/adaptation/resource_adaptation_processor.h"
#include "call/adaptation/resource_adaptation_processor_interface.h"
#include "call/adaptation/video_source_restrictions.h"
#include "call/adaptation/video_stream_input_state_provider.h"
@ -170,6 +171,8 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
DataRate stable_encoder_target;
};
class DegradationPreferenceManager;
void ReconfigureEncoder() RTC_RUN_ON(&encoder_queue_);
void OnEncoderSettingsChanged() RTC_RUN_ON(&encoder_queue_);
@ -415,9 +418,9 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
// Responsible for adapting input resolution or frame rate to ensure resources
// (e.g. CPU or bandwidth) are not overused.
// This class is single-threaded on the resource adaptation queue.
std::unique_ptr<ResourceAdaptationProcessorInterface>
resource_adaptation_processor_
RTC_GUARDED_BY(&resource_adaptation_queue_);
std::unique_ptr<ResourceAdaptationProcessor> resource_adaptation_processor_
RTC_GUARDED_BY(&resource_adaptation_queue_);
std::unique_ptr<DegradationPreferenceManager> degradation_preference_manager_;
std::vector<AdaptationConstraint*> adaptation_constraints_
RTC_GUARDED_BY(&resource_adaptation_queue_);
std::vector<AdaptationListener*> adaptation_listeners_