mirror of
https://github.com/mollyim/webrtc.git
synced 2025-05-13 05:40:42 +01:00
[Adaptation] VideoStreamAdapter broadcasts adaptations
This moves this responsibility from the ResourceAdaptaitonProcessor to the VideoStreamAdapter. A new interface for listening to adaptation changes was added, and the ResourceAdaptationProcessor now listens on the VideoStreamAdapter for those changes. This means that going forward, 1. We can do adaptations outside of resource limitations, like setting prior adaptations on a resource like initial frame dropper is designed to. 2. Adaptations can be on a different queue than the ResourceAdaptaitonProcessor's queue since updates are pushed through the listener. Bug: webrtc:11700 Change-Id: I6de0dec748dba095e702f0b9893c5afa50b51aa9 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/176859 Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org> Reviewed-by: Henrik Boström <hbos@webrtc.org> Commit-Queue: Evan Shrubsole <eshr@google.com> Cr-Commit-Position: refs/heads/master@{#31615}
This commit is contained in:
Evan Shrubsole
Commit Bot
parent
a1888ae791
commit
ec0af26ff8
13 changed files with 337 additions and 188 deletions
|
|
@ -15,6 +15,8 @@
|
|||
#include <utility>
|
||||
|
||||
#include "absl/algorithm/container.h"
|
||||
#include "api/video/video_adaptation_counters.h"
|
||||
#include "call/adaptation/video_stream_adapter.h"
|
||||
#include "rtc_base/logging.h"
|
||||
#include "rtc_base/ref_counted_object.h"
|
||||
#include "rtc_base/strings/string_builder.h"
|
||||
|
|
@ -67,7 +69,8 @@ ResourceAdaptationProcessor::MitigationResultAndLogMessage::
|
|||
|
||||
ResourceAdaptationProcessor::ResourceAdaptationProcessor(
|
||||
VideoStreamInputStateProvider* input_state_provider,
|
||||
VideoStreamEncoderObserver* encoder_stats_observer)
|
||||
VideoStreamEncoderObserver* encoder_stats_observer,
|
||||
VideoStreamAdapter* stream_adapter)
|
||||
: resource_adaptation_queue_(nullptr),
|
||||
resource_listener_delegate_(
|
||||
new rtc::RefCountedObject<ResourceListenerDelegate>(this)),
|
||||
|
|
@ -77,16 +80,15 @@ ResourceAdaptationProcessor::ResourceAdaptationProcessor(
|
|||
degradation_preference_(DegradationPreference::DISABLED),
|
||||
effective_degradation_preference_(DegradationPreference::DISABLED),
|
||||
is_screenshare_(false),
|
||||
stream_adapter_(std::make_unique<VideoStreamAdapter>()),
|
||||
stream_adapter_(stream_adapter),
|
||||
last_reported_source_restrictions_(),
|
||||
previous_mitigation_results_(),
|
||||
processing_in_progress_(false) {}
|
||||
processing_in_progress_(false) {
|
||||
RTC_DCHECK(stream_adapter_);
|
||||
}
|
||||
|
||||
ResourceAdaptationProcessor::~ResourceAdaptationProcessor() {
|
||||
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
|
||||
RTC_DCHECK(restrictions_listeners_.empty())
|
||||
<< "There are restrictions listener(s) depending on a "
|
||||
<< "ResourceAdaptationProcessor being destroyed.";
|
||||
RTC_DCHECK(resources_.empty())
|
||||
<< "There are resource(s) attached to a ResourceAdaptationProcessor "
|
||||
<< "being destroyed.";
|
||||
|
|
@ -96,6 +98,7 @@ ResourceAdaptationProcessor::~ResourceAdaptationProcessor() {
|
|||
RTC_DCHECK(adaptation_listeners_.empty())
|
||||
<< "There are listener(s) attached to a ResourceAdaptationProcessor "
|
||||
<< "being destroyed.";
|
||||
stream_adapter_->RemoveRestrictionsListener(this);
|
||||
resource_listener_delegate_->OnProcessorDestroyed();
|
||||
}
|
||||
|
||||
|
|
@ -107,6 +110,8 @@ void ResourceAdaptationProcessor::SetResourceAdaptationQueue(
|
|||
resource_listener_delegate_->SetResourceAdaptationQueue(
|
||||
resource_adaptation_queue);
|
||||
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
|
||||
// Now that we have the adaptation queue we can attach as adaptation listener.
|
||||
stream_adapter_->AddRestrictionsListener(this);
|
||||
}
|
||||
|
||||
DegradationPreference ResourceAdaptationProcessor::degradation_preference()
|
||||
|
|
@ -121,22 +126,23 @@ ResourceAdaptationProcessor::effective_degradation_preference() const {
|
|||
return effective_degradation_preference_;
|
||||
}
|
||||
|
||||
void ResourceAdaptationProcessor::AddRestrictionsListener(
|
||||
VideoSourceRestrictionsListener* restrictions_listener) {
|
||||
void ResourceAdaptationProcessor::AddResourceLimitationsListener(
|
||||
ResourceLimitationsListener* limitations_listener) {
|
||||
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
|
||||
RTC_DCHECK(std::find(restrictions_listeners_.begin(),
|
||||
restrictions_listeners_.end(),
|
||||
restrictions_listener) == restrictions_listeners_.end());
|
||||
restrictions_listeners_.push_back(restrictions_listener);
|
||||
RTC_DCHECK(std::find(resource_limitations_listeners_.begin(),
|
||||
resource_limitations_listeners_.end(),
|
||||
limitations_listener) ==
|
||||
resource_limitations_listeners_.end());
|
||||
resource_limitations_listeners_.push_back(limitations_listener);
|
||||
}
|
||||
|
||||
void ResourceAdaptationProcessor::RemoveRestrictionsListener(
|
||||
VideoSourceRestrictionsListener* restrictions_listener) {
|
||||
void ResourceAdaptationProcessor::RemoveResourceLimitationsListener(
|
||||
ResourceLimitationsListener* limitations_listener) {
|
||||
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
|
||||
auto it = std::find(restrictions_listeners_.begin(),
|
||||
restrictions_listeners_.end(), restrictions_listener);
|
||||
RTC_DCHECK(it != restrictions_listeners_.end());
|
||||
restrictions_listeners_.erase(it);
|
||||
auto it =
|
||||
std::find(resource_limitations_listeners_.begin(),
|
||||
resource_limitations_listeners_.end(), limitations_listener);
|
||||
RTC_DCHECK(it != resource_limitations_listeners_.end());
|
||||
resource_limitations_listeners_.erase(it);
|
||||
}
|
||||
|
||||
void ResourceAdaptationProcessor::AddResource(
|
||||
|
|
@ -232,39 +238,6 @@ void ResourceAdaptationProcessor::MaybeUpdateEffectiveDegradationPreference() {
|
|||
? DegradationPreference::MAINTAIN_RESOLUTION
|
||||
: degradation_preference_;
|
||||
stream_adapter_->SetDegradationPreference(effective_degradation_preference_);
|
||||
MaybeUpdateVideoSourceRestrictions(nullptr);
|
||||
}
|
||||
|
||||
void ResourceAdaptationProcessor::ResetVideoSourceRestrictions() {
|
||||
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
|
||||
RTC_LOG(INFO) << "Resetting restrictions";
|
||||
stream_adapter_->ClearRestrictions();
|
||||
adaptation_limits_by_resources_.clear();
|
||||
for (auto restrictions_listener : restrictions_listeners_) {
|
||||
restrictions_listener->OnResourceLimitationChanged(nullptr, {});
|
||||
}
|
||||
MaybeUpdateVideoSourceRestrictions(nullptr);
|
||||
}
|
||||
|
||||
void ResourceAdaptationProcessor::MaybeUpdateVideoSourceRestrictions(
|
||||
rtc::scoped_refptr<Resource> reason) {
|
||||
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
|
||||
VideoSourceRestrictions new_source_restrictions =
|
||||
FilterRestrictionsByDegradationPreference(
|
||||
stream_adapter_->source_restrictions(),
|
||||
effective_degradation_preference_);
|
||||
if (last_reported_source_restrictions_ != new_source_restrictions) {
|
||||
RTC_LOG(INFO) << "Reporting new restrictions (in "
|
||||
<< DegradationPreferenceToString(
|
||||
effective_degradation_preference_)
|
||||
<< "): " << new_source_restrictions.ToString();
|
||||
last_reported_source_restrictions_ = std::move(new_source_restrictions);
|
||||
for (auto* restrictions_listener : restrictions_listeners_) {
|
||||
restrictions_listener->OnVideoSourceRestrictionsUpdated(
|
||||
last_reported_source_restrictions_,
|
||||
stream_adapter_->adaptation_counters(), reason);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void ResourceAdaptationProcessor::OnResourceUsageStateMeasured(
|
||||
|
|
@ -386,10 +359,10 @@ ResourceAdaptationProcessor::OnResourceUnderuse(
|
|||
MitigationResult::kNotMostLimitedResource, message.Release());
|
||||
}
|
||||
|
||||
UpdateResourceLimitations(reason_resource, peek_restrictions);
|
||||
if (most_limited_resources.size() > 1) {
|
||||
// If there are multiple most limited resources, all must signal underuse
|
||||
// before the adaptation is applied.
|
||||
UpdateResourceLimitations(reason_resource, peek_restrictions);
|
||||
processing_in_progress_ = false;
|
||||
rtc::StringBuilder message;
|
||||
message << "Resource \"" << reason_resource->Name()
|
||||
|
|
@ -399,14 +372,11 @@ ResourceAdaptationProcessor::OnResourceUnderuse(
|
|||
}
|
||||
}
|
||||
// Apply adaptation.
|
||||
stream_adapter_->ApplyAdaptation(adaptation);
|
||||
stream_adapter_->ApplyAdaptation(adaptation, reason_resource);
|
||||
for (auto* adaptation_listener : adaptation_listeners_) {
|
||||
adaptation_listener->OnAdaptationApplied(
|
||||
input_state, restrictions_before, restrictions_after, reason_resource);
|
||||
}
|
||||
// Update VideoSourceRestrictions based on adaptation. This also informs the
|
||||
// |restrictions_listeners_|.
|
||||
MaybeUpdateVideoSourceRestrictions(reason_resource);
|
||||
processing_in_progress_ = false;
|
||||
rtc::StringBuilder message;
|
||||
message << "Adapted up successfully. Unfiltered adaptations: "
|
||||
|
|
@ -456,15 +426,11 @@ ResourceAdaptationProcessor::OnResourceOveruse(
|
|||
stream_adapter_->PeekNextRestrictions(adaptation);
|
||||
VideoSourceRestrictions restrictions_after =
|
||||
peek_next_restrictions.restrictions;
|
||||
UpdateResourceLimitations(reason_resource, peek_next_restrictions);
|
||||
stream_adapter_->ApplyAdaptation(adaptation);
|
||||
stream_adapter_->ApplyAdaptation(adaptation, reason_resource);
|
||||
for (auto* adaptation_listener : adaptation_listeners_) {
|
||||
adaptation_listener->OnAdaptationApplied(
|
||||
input_state, restrictions_before, restrictions_after, reason_resource);
|
||||
}
|
||||
// Update VideoSourceRestrictions based on adaptation. This also informs the
|
||||
// |restrictions_listeners_|.
|
||||
MaybeUpdateVideoSourceRestrictions(reason_resource);
|
||||
processing_in_progress_ = false;
|
||||
rtc::StringBuilder message;
|
||||
message << "Adapted down successfully. Unfiltered adaptations: "
|
||||
|
|
@ -520,18 +486,20 @@ ResourceAdaptationProcessor::FindMostLimitedResources() const {
|
|||
|
||||
void ResourceAdaptationProcessor::UpdateResourceLimitations(
|
||||
rtc::scoped_refptr<Resource> reason_resource,
|
||||
const VideoStreamAdapter::RestrictionsWithCounters&
|
||||
peek_next_restrictions) {
|
||||
adaptation_limits_by_resources_[reason_resource] = peek_next_restrictions;
|
||||
const VideoStreamAdapter::RestrictionsWithCounters& restrictions) {
|
||||
auto& adaptation_limits = adaptation_limits_by_resources_[reason_resource];
|
||||
if (adaptation_limits == restrictions) {
|
||||
return;
|
||||
}
|
||||
adaptation_limits = restrictions;
|
||||
|
||||
std::map<rtc::scoped_refptr<Resource>, VideoAdaptationCounters> limitations;
|
||||
for (const auto& p : adaptation_limits_by_resources_) {
|
||||
limitations.insert(std::make_pair(p.first, p.second.adaptation_counters));
|
||||
}
|
||||
|
||||
for (auto restrictions_listener : restrictions_listeners_) {
|
||||
restrictions_listener->OnResourceLimitationChanged(reason_resource,
|
||||
limitations);
|
||||
for (auto limitations_listener : resource_limitations_listeners_) {
|
||||
limitations_listener->OnResourceLimitationChanged(reason_resource,
|
||||
limitations);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -539,8 +507,8 @@ void ResourceAdaptationProcessor::
|
|||
MaybeUpdateResourceLimitationsOnResourceRemoval(
|
||||
VideoStreamAdapter::RestrictionsWithCounters removed_limitations) {
|
||||
if (adaptation_limits_by_resources_.empty()) {
|
||||
// Only the resource being removed was adapted so reset restrictions.
|
||||
ResetVideoSourceRestrictions();
|
||||
// Only the resource being removed was adapted so clear restrictions.
|
||||
stream_adapter_->ClearRestrictions();
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -558,20 +526,37 @@ void ResourceAdaptationProcessor::
|
|||
Adaptation adapt_to = stream_adapter_->GetAdaptationTo(
|
||||
most_limited.adaptation_counters, most_limited.restrictions);
|
||||
RTC_DCHECK_EQ(adapt_to.status(), Adaptation::Status::kValid);
|
||||
stream_adapter_->ApplyAdaptation(adapt_to);
|
||||
|
||||
RTC_LOG(INFO) << "Most limited resource removed. Restoring restrictions to "
|
||||
"next most limited restrictions: "
|
||||
<< most_limited.restrictions.ToString() << " with counters "
|
||||
<< most_limited.adaptation_counters.ToString();
|
||||
|
||||
MaybeUpdateVideoSourceRestrictions(nullptr);
|
||||
stream_adapter_->ApplyAdaptation(adapt_to, nullptr);
|
||||
auto input_state = input_state_provider_->InputState();
|
||||
for (auto* adaptation_listener : adaptation_listeners_) {
|
||||
adaptation_listener->OnAdaptationApplied(
|
||||
input_state, removed_limitations.restrictions,
|
||||
most_limited.restrictions, nullptr);
|
||||
}
|
||||
|
||||
RTC_LOG(INFO) << "Most limited resource removed. Restoring restrictions to "
|
||||
"next most limited restrictions: "
|
||||
<< most_limited.restrictions.ToString() << " with counters "
|
||||
<< most_limited.adaptation_counters.ToString();
|
||||
}
|
||||
|
||||
void ResourceAdaptationProcessor::OnVideoSourceRestrictionsUpdated(
|
||||
VideoSourceRestrictions restrictions,
|
||||
const VideoAdaptationCounters& adaptation_counters,
|
||||
rtc::scoped_refptr<Resource> reason,
|
||||
const VideoSourceRestrictions& unfiltered_restrictions) {
|
||||
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
|
||||
if (reason) {
|
||||
UpdateResourceLimitations(reason,
|
||||
{unfiltered_restrictions, adaptation_counters});
|
||||
} else if (adaptation_counters.Total() == 0) {
|
||||
// Adaptations are cleared.
|
||||
adaptation_limits_by_resources_.clear();
|
||||
previous_mitigation_results_.clear();
|
||||
for (auto limitations_listener : resource_limitations_listeners_) {
|
||||
limitations_listener->OnResourceLimitationChanged(nullptr, {});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
} // namespace webrtc
|
||||
|
|
|
|||
|
|
@ -51,11 +51,13 @@ namespace webrtc {
|
|||
// any thread but MUST subsequently be used and destroyed on a single sequence,
|
||||
// i.e. the "resource adaptation task queue".
|
||||
class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface,
|
||||
public VideoSourceRestrictionsListener,
|
||||
public ResourceListener {
|
||||
public:
|
||||
ResourceAdaptationProcessor(
|
||||
VideoStreamInputStateProvider* input_state_provider,
|
||||
VideoStreamEncoderObserver* encoder_stats_observer);
|
||||
VideoStreamEncoderObserver* encoder_stats_observer,
|
||||
VideoStreamAdapter* video_stream_adapter);
|
||||
~ResourceAdaptationProcessor() override;
|
||||
|
||||
void SetResourceAdaptationQueue(
|
||||
|
|
@ -65,10 +67,10 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface,
|
|||
DegradationPreference degradation_preference() const override;
|
||||
DegradationPreference effective_degradation_preference() const override;
|
||||
|
||||
void AddRestrictionsListener(
|
||||
VideoSourceRestrictionsListener* restrictions_listener) override;
|
||||
void RemoveRestrictionsListener(
|
||||
VideoSourceRestrictionsListener* restrictions_listener) override;
|
||||
void AddResourceLimitationsListener(
|
||||
ResourceLimitationsListener* limitations_listener) override;
|
||||
void RemoveResourceLimitationsListener(
|
||||
ResourceLimitationsListener* limitations_listener) override;
|
||||
void AddResource(rtc::scoped_refptr<Resource> resource) override;
|
||||
std::vector<rtc::scoped_refptr<Resource>> GetResources() const override;
|
||||
void RemoveResource(rtc::scoped_refptr<Resource> resource) override;
|
||||
|
|
@ -83,13 +85,19 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface,
|
|||
void SetDegradationPreference(
|
||||
DegradationPreference degradation_preference) override;
|
||||
void SetIsScreenshare(bool is_screenshare) override;
|
||||
void ResetVideoSourceRestrictions() override;
|
||||
|
||||
// ResourceListener implementation.
|
||||
// Triggers OnResourceUnderuse() or OnResourceOveruse().
|
||||
void OnResourceUsageStateMeasured(rtc::scoped_refptr<Resource> resource,
|
||||
ResourceUsageState usage_state) override;
|
||||
|
||||
// VideoSourceRestrictionsListener implementation.
|
||||
void OnVideoSourceRestrictionsUpdated(
|
||||
VideoSourceRestrictions restrictions,
|
||||
const VideoAdaptationCounters& adaptation_counters,
|
||||
rtc::scoped_refptr<Resource> reason,
|
||||
const VideoSourceRestrictions& unfiltered_restrictions) override;
|
||||
|
||||
// May trigger 1-2 adaptations. It is meant to reduce resolution but this is
|
||||
// not guaranteed. It may adapt frame rate, which does not address the issue.
|
||||
// TODO(hbos): Can we get rid of this?
|
||||
|
|
@ -149,9 +157,6 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface,
|
|||
// Needs to be invoked any time |degradation_preference_| or |is_screenshare_|
|
||||
// changes to ensure |effective_degradation_preference_| is up-to-date.
|
||||
void MaybeUpdateEffectiveDegradationPreference();
|
||||
// If the filtered source restrictions are different than
|
||||
// |last_reported_source_restrictions_|, inform the listeners.
|
||||
void MaybeUpdateVideoSourceRestrictions(rtc::scoped_refptr<Resource> reason);
|
||||
|
||||
void UpdateResourceLimitations(
|
||||
rtc::scoped_refptr<Resource> reason_resource,
|
||||
|
|
@ -178,7 +183,7 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface,
|
|||
RTC_GUARDED_BY(resource_adaptation_queue_);
|
||||
VideoStreamEncoderObserver* const encoder_stats_observer_
|
||||
RTC_GUARDED_BY(resource_adaptation_queue_);
|
||||
std::vector<VideoSourceRestrictionsListener*> restrictions_listeners_
|
||||
std::vector<ResourceLimitationsListener*> resource_limitations_listeners_
|
||||
RTC_GUARDED_BY(resource_adaptation_queue_);
|
||||
std::vector<rtc::scoped_refptr<Resource>> resources_
|
||||
RTC_GUARDED_BY(resource_adaptation_queue_);
|
||||
|
|
@ -198,7 +203,7 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface,
|
|||
RTC_GUARDED_BY(resource_adaptation_queue_);
|
||||
bool is_screenshare_ RTC_GUARDED_BY(resource_adaptation_queue_);
|
||||
// Responsible for generating and applying possible adaptations.
|
||||
const std::unique_ptr<VideoStreamAdapter> stream_adapter_
|
||||
VideoStreamAdapter* const stream_adapter_
|
||||
RTC_GUARDED_BY(resource_adaptation_queue_);
|
||||
VideoSourceRestrictions last_reported_source_restrictions_
|
||||
RTC_GUARDED_BY(resource_adaptation_queue_);
|
||||
|
|
|
|||
|
|
@ -12,9 +12,9 @@
|
|||
|
||||
namespace webrtc {
|
||||
|
||||
VideoSourceRestrictionsListener::~VideoSourceRestrictionsListener() = default;
|
||||
|
||||
ResourceAdaptationProcessorInterface::~ResourceAdaptationProcessorInterface() =
|
||||
default;
|
||||
|
||||
ResourceLimitationsListener::~ResourceLimitationsListener() = default;
|
||||
|
||||
} // namespace webrtc
|
||||
|
|
|
|||
|
|
@ -28,26 +28,16 @@
|
|||
|
||||
namespace webrtc {
|
||||
|
||||
// The listener is responsible for carrying out the reconfiguration of the video
|
||||
// source such that the VideoSourceRestrictions are fulfilled.
|
||||
class VideoSourceRestrictionsListener {
|
||||
class ResourceLimitationsListener {
|
||||
public:
|
||||
virtual ~VideoSourceRestrictionsListener();
|
||||
|
||||
// The |restrictions| are filtered by degradation preference but not the
|
||||
// |adaptation_counters|, which are currently only reported for legacy stats
|
||||
// calculation purposes.
|
||||
virtual void OnVideoSourceRestrictionsUpdated(
|
||||
VideoSourceRestrictions restrictions,
|
||||
const VideoAdaptationCounters& adaptation_counters,
|
||||
rtc::scoped_refptr<Resource> reason) = 0;
|
||||
virtual ~ResourceLimitationsListener();
|
||||
|
||||
// The limitations on a resource were changed. This does not mean the current
|
||||
// video restrictions have changed.
|
||||
virtual void OnResourceLimitationChanged(
|
||||
rtc::scoped_refptr<Resource> resource,
|
||||
const std::map<rtc::scoped_refptr<Resource>, VideoAdaptationCounters>&
|
||||
resource_limitations) {}
|
||||
resource_limitations) = 0;
|
||||
};
|
||||
|
||||
// The Resource Adaptation Processor is responsible for reacting to resource
|
||||
|
|
@ -74,10 +64,10 @@ class ResourceAdaptationProcessorInterface {
|
|||
// with AddResource() and RemoveResource() instead. When the processor is
|
||||
// multi-stream aware, stream-specific resouces will get added and removed
|
||||
// over time.
|
||||
virtual void AddRestrictionsListener(
|
||||
VideoSourceRestrictionsListener* restrictions_listener) = 0;
|
||||
virtual void RemoveRestrictionsListener(
|
||||
VideoSourceRestrictionsListener* restrictions_listener) = 0;
|
||||
virtual void AddResourceLimitationsListener(
|
||||
ResourceLimitationsListener* limitations_listener) = 0;
|
||||
virtual void RemoveResourceLimitationsListener(
|
||||
ResourceLimitationsListener* limitations_listener) = 0;
|
||||
virtual void AddResource(rtc::scoped_refptr<Resource> resource) = 0;
|
||||
virtual std::vector<rtc::scoped_refptr<Resource>> GetResources() const = 0;
|
||||
virtual void RemoveResource(rtc::scoped_refptr<Resource> resource) = 0;
|
||||
|
|
@ -93,7 +83,6 @@ class ResourceAdaptationProcessorInterface {
|
|||
virtual void SetDegradationPreference(
|
||||
DegradationPreference degradation_preference) = 0;
|
||||
virtual void SetIsScreenshare(bool is_screenshare) = 0;
|
||||
virtual void ResetVideoSourceRestrictions() = 0;
|
||||
|
||||
// May trigger one or more adaptations. It is meant to reduce resolution -
|
||||
// useful if a frame was dropped due to its size - however, the implementation
|
||||
|
|
|
|||
|
|
@ -65,7 +65,8 @@ class VideoSourceRestrictionsListenerForTesting
|
|||
void OnVideoSourceRestrictionsUpdated(
|
||||
VideoSourceRestrictions restrictions,
|
||||
const VideoAdaptationCounters& adaptation_counters,
|
||||
rtc::scoped_refptr<Resource> reason) override {
|
||||
rtc::scoped_refptr<Resource> reason,
|
||||
const VideoSourceRestrictions& unfiltered_restrictions) override {
|
||||
RTC_DCHECK_RUN_ON(&sequence_checker_);
|
||||
++restrictions_updated_count_;
|
||||
restrictions_ = restrictions;
|
||||
|
|
@ -91,11 +92,13 @@ class ResourceAdaptationProcessorTest : public ::testing::Test {
|
|||
other_resource_(FakeResource::Create("OtherFakeResource")),
|
||||
adaptation_constraint_("FakeAdaptationConstraint"),
|
||||
adaptation_listener_(),
|
||||
video_stream_adapter_(std::make_unique<VideoStreamAdapter>()),
|
||||
processor_(std::make_unique<ResourceAdaptationProcessor>(
|
||||
&input_state_provider_,
|
||||
/*encoder_stats_observer=*/&frame_rate_provider_)) {
|
||||
/*encoder_stats_observer=*/&frame_rate_provider_,
|
||||
video_stream_adapter_.get())) {
|
||||
processor_->SetResourceAdaptationQueue(TaskQueueBase::Current());
|
||||
processor_->AddRestrictionsListener(&restrictions_listener_);
|
||||
video_stream_adapter_->AddRestrictionsListener(&restrictions_listener_);
|
||||
processor_->AddResource(resource_);
|
||||
processor_->AddResource(other_resource_);
|
||||
processor_->AddAdaptationConstraint(&adaptation_constraint_);
|
||||
|
|
@ -122,7 +125,6 @@ class ResourceAdaptationProcessorTest : public ::testing::Test {
|
|||
}
|
||||
|
||||
void DestroyProcessor() {
|
||||
processor_->RemoveRestrictionsListener(&restrictions_listener_);
|
||||
if (resource_) {
|
||||
processor_->RemoveResource(resource_);
|
||||
}
|
||||
|
|
@ -131,6 +133,7 @@ class ResourceAdaptationProcessorTest : public ::testing::Test {
|
|||
}
|
||||
processor_->RemoveAdaptationConstraint(&adaptation_constraint_);
|
||||
processor_->RemoveAdaptationListener(&adaptation_listener_);
|
||||
video_stream_adapter_->RemoveRestrictionsListener(&restrictions_listener_);
|
||||
processor_.reset();
|
||||
}
|
||||
|
||||
|
|
@ -145,6 +148,7 @@ class ResourceAdaptationProcessorTest : public ::testing::Test {
|
|||
rtc::scoped_refptr<FakeResource> other_resource_;
|
||||
FakeAdaptationConstraint adaptation_constraint_;
|
||||
FakeAdaptationListener adaptation_listener_;
|
||||
std::unique_ptr<VideoStreamAdapter> video_stream_adapter_;
|
||||
std::unique_ptr<ResourceAdaptationProcessor> processor_;
|
||||
VideoSourceRestrictionsListenerForTesting restrictions_listener_;
|
||||
};
|
||||
|
|
@ -290,7 +294,7 @@ TEST_F(ResourceAdaptationProcessorTest,
|
|||
resource_->SetUsageState(ResourceUsageState::kOveruse);
|
||||
EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count());
|
||||
|
||||
processor_->ResetVideoSourceRestrictions();
|
||||
video_stream_adapter_->ClearRestrictions();
|
||||
EXPECT_EQ(0, restrictions_listener_.adaptation_counters().Total());
|
||||
other_resource_->SetUsageState(ResourceUsageState::kOveruse);
|
||||
EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total());
|
||||
|
|
|
|||
|
|
@ -61,6 +61,8 @@ int GetLowerResolutionThan(int pixel_count) {
|
|||
|
||||
} // namespace
|
||||
|
||||
VideoSourceRestrictionsListener::~VideoSourceRestrictionsListener() = default;
|
||||
|
||||
VideoSourceRestrictions FilterRestrictionsByDegradationPreference(
|
||||
VideoSourceRestrictions source_restrictions,
|
||||
DegradationPreference degradation_preference) {
|
||||
|
|
@ -82,28 +84,6 @@ VideoSourceRestrictions FilterRestrictionsByDegradationPreference(
|
|||
return source_restrictions;
|
||||
}
|
||||
|
||||
VideoAdaptationCounters FilterVideoAdaptationCountersByDegradationPreference(
|
||||
VideoAdaptationCounters counters,
|
||||
DegradationPreference degradation_preference) {
|
||||
switch (degradation_preference) {
|
||||
case DegradationPreference::BALANCED:
|
||||
break;
|
||||
case DegradationPreference::MAINTAIN_FRAMERATE:
|
||||
counters.fps_adaptations = 0;
|
||||
break;
|
||||
case DegradationPreference::MAINTAIN_RESOLUTION:
|
||||
counters.resolution_adaptations = 0;
|
||||
break;
|
||||
case DegradationPreference::DISABLED:
|
||||
counters.resolution_adaptations = 0;
|
||||
counters.fps_adaptations = 0;
|
||||
break;
|
||||
default:
|
||||
RTC_NOTREACHED();
|
||||
}
|
||||
return counters;
|
||||
}
|
||||
|
||||
// TODO(hbos): Use absl::optional<> instead?
|
||||
int GetHigherResolutionThan(int pixel_count) {
|
||||
return pixel_count != std::numeric_limits<int>::max()
|
||||
|
|
@ -355,40 +335,73 @@ VideoStreamAdapter::VideoStreamAdapter()
|
|||
adaptation_validation_id_(0),
|
||||
degradation_preference_(DegradationPreference::DISABLED),
|
||||
input_state_(),
|
||||
last_adaptation_request_(absl::nullopt) {}
|
||||
last_adaptation_request_(absl::nullopt),
|
||||
last_video_source_restrictions_() {
|
||||
sequence_checker_.Detach();
|
||||
}
|
||||
|
||||
VideoStreamAdapter::~VideoStreamAdapter() {}
|
||||
|
||||
VideoSourceRestrictions VideoStreamAdapter::source_restrictions() const {
|
||||
RTC_DCHECK_RUN_ON(&sequence_checker_);
|
||||
return source_restrictor_->source_restrictions();
|
||||
}
|
||||
|
||||
const VideoAdaptationCounters& VideoStreamAdapter::adaptation_counters() const {
|
||||
RTC_DCHECK_RUN_ON(&sequence_checker_);
|
||||
return source_restrictor_->adaptation_counters();
|
||||
}
|
||||
|
||||
void VideoStreamAdapter::ClearRestrictions() {
|
||||
RTC_DCHECK_RUN_ON(&sequence_checker_);
|
||||
// Invalidate any previously returned Adaptation.
|
||||
RTC_LOG(INFO) << "Resetting restrictions";
|
||||
++adaptation_validation_id_;
|
||||
source_restrictor_->ClearRestrictions();
|
||||
last_adaptation_request_.reset();
|
||||
BroadcastVideoRestrictionsUpdate(nullptr);
|
||||
}
|
||||
|
||||
void VideoStreamAdapter::AddRestrictionsListener(
|
||||
VideoSourceRestrictionsListener* restrictions_listener) {
|
||||
RTC_DCHECK_RUN_ON(&sequence_checker_);
|
||||
RTC_DCHECK(std::find(restrictions_listeners_.begin(),
|
||||
restrictions_listeners_.end(),
|
||||
restrictions_listener) == restrictions_listeners_.end());
|
||||
restrictions_listeners_.push_back(restrictions_listener);
|
||||
}
|
||||
|
||||
void VideoStreamAdapter::RemoveRestrictionsListener(
|
||||
VideoSourceRestrictionsListener* restrictions_listener) {
|
||||
RTC_DCHECK_RUN_ON(&sequence_checker_);
|
||||
auto it = std::find(restrictions_listeners_.begin(),
|
||||
restrictions_listeners_.end(), restrictions_listener);
|
||||
RTC_DCHECK(it != restrictions_listeners_.end());
|
||||
restrictions_listeners_.erase(it);
|
||||
}
|
||||
|
||||
void VideoStreamAdapter::SetDegradationPreference(
|
||||
DegradationPreference degradation_preference) {
|
||||
RTC_DCHECK_RUN_ON(&sequence_checker_);
|
||||
if (degradation_preference_ == degradation_preference)
|
||||
return;
|
||||
// Invalidate any previously returned Adaptation.
|
||||
++adaptation_validation_id_;
|
||||
if (degradation_preference == DegradationPreference::BALANCED ||
|
||||
degradation_preference_ == DegradationPreference::BALANCED) {
|
||||
ClearRestrictions();
|
||||
}
|
||||
bool balanced_switch =
|
||||
degradation_preference == DegradationPreference::BALANCED ||
|
||||
degradation_preference_ == DegradationPreference::BALANCED;
|
||||
degradation_preference_ = degradation_preference;
|
||||
if (balanced_switch) {
|
||||
// ClearRestrictions() calls BroadcastVideoRestrictionsUpdate(nullptr).
|
||||
ClearRestrictions();
|
||||
} else {
|
||||
BroadcastVideoRestrictionsUpdate(nullptr);
|
||||
}
|
||||
}
|
||||
|
||||
void VideoStreamAdapter::SetInput(VideoStreamInputState input_state) {
|
||||
// Invalidate any previously returned Adaptation.
|
||||
RTC_DCHECK_RUN_ON(&sequence_checker_);
|
||||
++adaptation_validation_id_;
|
||||
input_state_ = input_state;
|
||||
source_restrictor_->set_min_pixels_per_frame(
|
||||
|
|
@ -396,6 +409,7 @@ void VideoStreamAdapter::SetInput(VideoStreamInputState input_state) {
|
|||
}
|
||||
|
||||
Adaptation VideoStreamAdapter::GetAdaptationUp() const {
|
||||
RTC_DCHECK_RUN_ON(&sequence_checker_);
|
||||
RTC_DCHECK_NE(degradation_preference_, DegradationPreference::DISABLED);
|
||||
RTC_DCHECK(input_state_.HasInputFrameSizeAndFramesPerSecond());
|
||||
// Don't adapt if we're awaiting a previous adaptation to have an effect.
|
||||
|
|
@ -469,6 +483,7 @@ Adaptation VideoStreamAdapter::GetAdaptationUp() const {
|
|||
}
|
||||
|
||||
Adaptation VideoStreamAdapter::GetAdaptationDown() const {
|
||||
RTC_DCHECK_RUN_ON(&sequence_checker_);
|
||||
RTC_DCHECK_NE(degradation_preference_, DegradationPreference::DISABLED);
|
||||
RTC_DCHECK(input_state_.HasInputFrameSizeAndFramesPerSecond());
|
||||
// Don't adapt if we're awaiting a previous adaptation to have an effect or
|
||||
|
|
@ -537,11 +552,13 @@ Adaptation VideoStreamAdapter::GetAdaptationDown() const {
|
|||
|
||||
VideoStreamAdapter::RestrictionsWithCounters
|
||||
VideoStreamAdapter::PeekNextRestrictions(const Adaptation& adaptation) const {
|
||||
RTC_DCHECK_RUN_ON(&sequence_checker_);
|
||||
RTC_DCHECK_EQ(adaptation.validation_id_, adaptation_validation_id_);
|
||||
RTC_LOG(LS_INFO) << "PeekNextRestrictions called";
|
||||
if (adaptation.status() != Adaptation::Status::kValid)
|
||||
if (adaptation.status() != Adaptation::Status::kValid) {
|
||||
return {source_restrictor_->source_restrictions(),
|
||||
source_restrictor_->adaptation_counters()};
|
||||
}
|
||||
VideoSourceRestrictor restrictor_copy = *source_restrictor_;
|
||||
restrictor_copy.ApplyAdaptationStep(adaptation.step(),
|
||||
degradation_preference_);
|
||||
|
|
@ -549,7 +566,10 @@ VideoStreamAdapter::PeekNextRestrictions(const Adaptation& adaptation) const {
|
|||
restrictor_copy.adaptation_counters()};
|
||||
}
|
||||
|
||||
void VideoStreamAdapter::ApplyAdaptation(const Adaptation& adaptation) {
|
||||
void VideoStreamAdapter::ApplyAdaptation(
|
||||
const Adaptation& adaptation,
|
||||
rtc::scoped_refptr<Resource> resource) {
|
||||
RTC_DCHECK_RUN_ON(&sequence_checker_);
|
||||
RTC_DCHECK_EQ(adaptation.validation_id_, adaptation_validation_id_);
|
||||
RTC_LOG(LS_INFO) << "ApplyAdaptation called";
|
||||
if (adaptation.status() != Adaptation::Status::kValid)
|
||||
|
|
@ -562,14 +582,33 @@ void VideoStreamAdapter::ApplyAdaptation(const Adaptation& adaptation) {
|
|||
// Adapt!
|
||||
source_restrictor_->ApplyAdaptationStep(adaptation.step(),
|
||||
degradation_preference_);
|
||||
BroadcastVideoRestrictionsUpdate(resource);
|
||||
}
|
||||
|
||||
Adaptation VideoStreamAdapter::GetAdaptationTo(
|
||||
const VideoAdaptationCounters& counters,
|
||||
const VideoSourceRestrictions& restrictions) const {
|
||||
// Adapts up/down from the current levels so counters are equal.
|
||||
RTC_DCHECK_RUN_ON(&sequence_checker_);
|
||||
return Adaptation(adaptation_validation_id_,
|
||||
Adaptation::Step(restrictions, counters));
|
||||
}
|
||||
|
||||
void VideoStreamAdapter::BroadcastVideoRestrictionsUpdate(
|
||||
const rtc::scoped_refptr<Resource>& resource) {
|
||||
RTC_DCHECK_RUN_ON(&sequence_checker_);
|
||||
VideoSourceRestrictions filtered = FilterRestrictionsByDegradationPreference(
|
||||
source_restrictions(), degradation_preference_);
|
||||
if (last_filtered_restrictions_ == filtered) {
|
||||
return;
|
||||
}
|
||||
for (auto* restrictions_listener : restrictions_listeners_) {
|
||||
restrictions_listener->OnVideoSourceRestrictionsUpdated(
|
||||
filtered, source_restrictor_->adaptation_counters(), resource,
|
||||
source_restrictions());
|
||||
}
|
||||
last_video_source_restrictions_ = source_restrictor_->source_restrictions();
|
||||
last_filtered_restrictions_ = filtered;
|
||||
}
|
||||
|
||||
} // namespace webrtc
|
||||
|
|
|
|||
|
|
@ -12,6 +12,7 @@
|
|||
#define CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_
|
||||
|
||||
#include <memory>
|
||||
#include <vector>
|
||||
|
||||
#include "absl/types/optional.h"
|
||||
#include "api/adaptation/resource.h"
|
||||
|
|
@ -24,6 +25,22 @@
|
|||
|
||||
namespace webrtc {
|
||||
|
||||
// The listener is responsible for carrying out the reconfiguration of the video
|
||||
// source such that the VideoSourceRestrictions are fulfilled.
|
||||
class VideoSourceRestrictionsListener {
|
||||
public:
|
||||
virtual ~VideoSourceRestrictionsListener();
|
||||
|
||||
// The |restrictions| are filtered by degradation preference but not the
|
||||
// |adaptation_counters|, which are currently only reported for legacy stats
|
||||
// calculation purposes.
|
||||
virtual void OnVideoSourceRestrictionsUpdated(
|
||||
VideoSourceRestrictions restrictions,
|
||||
const VideoAdaptationCounters& adaptation_counters,
|
||||
rtc::scoped_refptr<Resource> reason,
|
||||
const VideoSourceRestrictions& unfiltered_restrictions) = 0;
|
||||
};
|
||||
|
||||
class VideoStreamAdapter;
|
||||
|
||||
extern const int kMinFrameRateFps;
|
||||
|
|
@ -32,10 +49,6 @@ VideoSourceRestrictions FilterRestrictionsByDegradationPreference(
|
|||
VideoSourceRestrictions source_restrictions,
|
||||
DegradationPreference degradation_preference);
|
||||
|
||||
VideoAdaptationCounters FilterVideoAdaptationCountersByDegradationPreference(
|
||||
VideoAdaptationCounters counters,
|
||||
DegradationPreference degradation_preference);
|
||||
|
||||
int GetHigherResolutionThan(int pixel_count);
|
||||
|
||||
// Represents one step that the VideoStreamAdapter can take when adapting the
|
||||
|
|
@ -128,6 +141,11 @@ class VideoStreamAdapter {
|
|||
const VideoAdaptationCounters& adaptation_counters() const;
|
||||
void ClearRestrictions();
|
||||
|
||||
void AddRestrictionsListener(
|
||||
VideoSourceRestrictionsListener* restrictions_listener);
|
||||
void RemoveRestrictionsListener(
|
||||
VideoSourceRestrictionsListener* restrictions_listener);
|
||||
|
||||
// TODO(hbos): Setting the degradation preference should not clear
|
||||
// restrictions! This is not defined in the spec and is unexpected, there is a
|
||||
// tiny risk that people would discover and rely on this behavior.
|
||||
|
|
@ -145,6 +163,15 @@ class VideoStreamAdapter {
|
|||
struct RestrictionsWithCounters {
|
||||
VideoSourceRestrictions restrictions;
|
||||
VideoAdaptationCounters adaptation_counters;
|
||||
|
||||
bool operator==(const RestrictionsWithCounters& other) {
|
||||
return restrictions == other.restrictions &&
|
||||
adaptation_counters == other.adaptation_counters;
|
||||
}
|
||||
|
||||
bool operator!=(const RestrictionsWithCounters& other) {
|
||||
return !(*this == other);
|
||||
}
|
||||
};
|
||||
|
||||
// Returns the restrictions that result from applying the adaptation, without
|
||||
|
|
@ -152,12 +179,19 @@ class VideoStreamAdapter {
|
|||
// are returned.
|
||||
RestrictionsWithCounters PeekNextRestrictions(
|
||||
const Adaptation& adaptation) const;
|
||||
// Updates source_restrictions() based according to the Adaptation.
|
||||
void ApplyAdaptation(const Adaptation& adaptation);
|
||||
// Updates source_restrictions() based according to the Adaptation. These
|
||||
// adaptations will be attributed to the Resource |resource| if the |resource|
|
||||
// is non-null. If |resource| is null the adaptation will be changed in
|
||||
// general, and thus could be adapted up in the future from other resources.
|
||||
void ApplyAdaptation(const Adaptation& adaptation,
|
||||
rtc::scoped_refptr<Resource> resource);
|
||||
|
||||
private:
|
||||
class VideoSourceRestrictor;
|
||||
|
||||
void BroadcastVideoRestrictionsUpdate(
|
||||
const rtc::scoped_refptr<Resource>& resource);
|
||||
|
||||
// The input frame rate and resolution at the time of an adaptation in the
|
||||
// direction described by |mode_| (up or down).
|
||||
// TODO(https://crbug.com/webrtc/11393): Can this be renamed? Can this be
|
||||
|
|
@ -171,18 +205,21 @@ class VideoStreamAdapter {
|
|||
Adaptation::StepType step_type_;
|
||||
};
|
||||
|
||||
SequenceChecker sequence_checker_ RTC_GUARDED_BY(&sequence_checker_);
|
||||
// Owner and modifier of the VideoSourceRestriction of this stream adaptor.
|
||||
const std::unique_ptr<VideoSourceRestrictor> source_restrictor_;
|
||||
const std::unique_ptr<VideoSourceRestrictor> source_restrictor_
|
||||
RTC_GUARDED_BY(&sequence_checker_);
|
||||
// Decides the next adaptation target in DegradationPreference::BALANCED.
|
||||
const BalancedDegradationSettings balanced_settings_;
|
||||
// To guard against applying adaptations that have become invalidated, an
|
||||
// Adaptation that is applied has to have a matching validation ID.
|
||||
int adaptation_validation_id_;
|
||||
int adaptation_validation_id_ RTC_GUARDED_BY(&sequence_checker_);
|
||||
// When deciding the next target up or down, different strategies are used
|
||||
// depending on the DegradationPreference.
|
||||
// https://w3c.github.io/mst-content-hint/#dom-rtcdegradationpreference
|
||||
DegradationPreference degradation_preference_;
|
||||
VideoStreamInputState input_state_;
|
||||
DegradationPreference degradation_preference_
|
||||
RTC_GUARDED_BY(&sequence_checker_);
|
||||
VideoStreamInputState input_state_ RTC_GUARDED_BY(&sequence_checker_);
|
||||
// The input frame rate, resolution and adaptation direction of the last
|
||||
// ApplyAdaptationTarget(). Used to avoid adapting twice if a recent
|
||||
// adaptation has not had an effect on the input frame rate or resolution yet.
|
||||
|
|
@ -190,7 +227,16 @@ class VideoStreamAdapter {
|
|||
// resources intead? If we already have adapted it seems like we should wait
|
||||
// a while before adapting again, so that we are not acting on usage
|
||||
// measurements that are made obsolete/unreliable by an "ongoing" adaptation.
|
||||
absl::optional<AdaptationRequest> last_adaptation_request_;
|
||||
absl::optional<AdaptationRequest> last_adaptation_request_
|
||||
RTC_GUARDED_BY(&sequence_checker_);
|
||||
// The previous restrictions value. Starts as unrestricted.
|
||||
VideoSourceRestrictions last_video_source_restrictions_
|
||||
RTC_GUARDED_BY(&sequence_checker_);
|
||||
VideoSourceRestrictions last_filtered_restrictions_
|
||||
RTC_GUARDED_BY(&sequence_checker_);
|
||||
|
||||
std::vector<VideoSourceRestrictionsListener*> restrictions_listeners_
|
||||
RTC_GUARDED_BY(&sequence_checker_);
|
||||
};
|
||||
|
||||
} // namespace webrtc
|
||||
|
|
|
|||
|
|
@ -85,7 +85,7 @@ class FakeVideoStream {
|
|||
// Performs ApplyAdaptation() followed by SetInput() with input pixels and
|
||||
// frame rate adjusted according to the resulting restrictions.
|
||||
void ApplyAdaptation(Adaptation adaptation) {
|
||||
adapter_->ApplyAdaptation(adaptation);
|
||||
adapter_->ApplyAdaptation(adaptation, nullptr);
|
||||
// Update input pixels and fps according to the resulting restrictions.
|
||||
auto restrictions = adapter_->source_restrictions();
|
||||
if (restrictions.target_pixels_per_frame().has_value()) {
|
||||
|
|
@ -110,6 +110,28 @@ class FakeVideoStream {
|
|||
int min_pixels_per_frame_;
|
||||
};
|
||||
|
||||
class FakeVideoStreamAdapterListner : public VideoSourceRestrictionsListener {
|
||||
public:
|
||||
void OnVideoSourceRestrictionsUpdated(
|
||||
VideoSourceRestrictions restrictions,
|
||||
const VideoAdaptationCounters& adaptation_counters,
|
||||
rtc::scoped_refptr<Resource> reason,
|
||||
const VideoSourceRestrictions& unfiltered_restrictions) {
|
||||
calls_++;
|
||||
last_restrictions_ = unfiltered_restrictions;
|
||||
}
|
||||
|
||||
int calls() const { return calls_; }
|
||||
|
||||
VideoSourceRestrictions last_restrictions() const {
|
||||
return last_restrictions_;
|
||||
}
|
||||
|
||||
private:
|
||||
int calls_ = 0;
|
||||
VideoSourceRestrictions last_restrictions_;
|
||||
};
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST(VideoStreamAdapterTest, NoRestrictionsByDefault) {
|
||||
|
|
@ -126,7 +148,7 @@ TEST(VideoStreamAdapterTest, MaintainFramerate_DecreasesPixelsToThreeFifths) {
|
|||
Adaptation adaptation = adapter.GetAdaptationDown();
|
||||
EXPECT_EQ(Adaptation::Status::kValid, adaptation.status());
|
||||
EXPECT_FALSE(adaptation.min_pixel_limit_reached());
|
||||
adapter.ApplyAdaptation(adaptation);
|
||||
adapter.ApplyAdaptation(adaptation, nullptr);
|
||||
EXPECT_EQ(static_cast<size_t>((kInputPixels * 3) / 5),
|
||||
adapter.source_restrictions().max_pixels_per_frame());
|
||||
EXPECT_EQ(absl::nullopt,
|
||||
|
|
@ -195,7 +217,7 @@ TEST(VideoStreamAdapterTest, MaintainResolution_DecreasesFpsToTwoThirds) {
|
|||
InputState(1280 * 720, kInputFps, kDefaultMinPixelsPerFrame));
|
||||
Adaptation adaptation = adapter.GetAdaptationDown();
|
||||
EXPECT_EQ(Adaptation::Status::kValid, adaptation.status());
|
||||
adapter.ApplyAdaptation(adaptation);
|
||||
adapter.ApplyAdaptation(adaptation, nullptr);
|
||||
EXPECT_EQ(absl::nullopt,
|
||||
adapter.source_restrictions().max_pixels_per_frame());
|
||||
EXPECT_EQ(absl::nullopt,
|
||||
|
|
@ -275,7 +297,7 @@ TEST(VideoStreamAdapterTest, Balanced_DecreaseFrameRate) {
|
|||
// resolution: kBalancedMediumFrameRateFps.
|
||||
Adaptation adaptation = adapter.GetAdaptationDown();
|
||||
EXPECT_EQ(Adaptation::Status::kValid, adaptation.status());
|
||||
adapter.ApplyAdaptation(adaptation);
|
||||
adapter.ApplyAdaptation(adaptation, nullptr);
|
||||
EXPECT_EQ(absl::nullopt,
|
||||
adapter.source_restrictions().max_pixels_per_frame());
|
||||
EXPECT_EQ(absl::nullopt,
|
||||
|
|
@ -519,7 +541,7 @@ TEST(VideoStreamAdapterTest, MaintainFramerate_AwaitingPreviousAdaptationDown) {
|
|||
adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE);
|
||||
adapter.SetInput(InputState(1280 * 720, 30, kDefaultMinPixelsPerFrame));
|
||||
// Adapt down once, but don't update the input.
|
||||
adapter.ApplyAdaptation(adapter.GetAdaptationDown());
|
||||
adapter.ApplyAdaptation(adapter.GetAdaptationDown(), nullptr);
|
||||
EXPECT_EQ(1, adapter.adaptation_counters().resolution_adaptations);
|
||||
{
|
||||
// Having performed the adaptation, but not updated the input based on the
|
||||
|
|
@ -541,7 +563,7 @@ TEST(VideoStreamAdapterTest, MaintainFramerate_AwaitingPreviousAdaptationUp) {
|
|||
fake_stream.ApplyAdaptation(adapter.GetAdaptationDown());
|
||||
EXPECT_EQ(2, adapter.adaptation_counters().resolution_adaptations);
|
||||
// Adapt up once, but don't update the input.
|
||||
adapter.ApplyAdaptation(adapter.GetAdaptationUp());
|
||||
adapter.ApplyAdaptation(adapter.GetAdaptationUp(), nullptr);
|
||||
EXPECT_EQ(1, adapter.adaptation_counters().resolution_adaptations);
|
||||
{
|
||||
// Having performed the adaptation, but not updated the input based on the
|
||||
|
|
@ -614,7 +636,7 @@ TEST(VideoStreamAdapterTest,
|
|||
adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE);
|
||||
fake_stream.ApplyAdaptation(adapter.GetAdaptationDown());
|
||||
// Apply adaptation up but don't update input.
|
||||
adapter.ApplyAdaptation(adapter.GetAdaptationUp());
|
||||
adapter.ApplyAdaptation(adapter.GetAdaptationUp(), nullptr);
|
||||
EXPECT_EQ(Adaptation::Status::kAwaitingPreviousAdaptation,
|
||||
adapter.GetAdaptationUp().status());
|
||||
|
||||
|
|
@ -668,7 +690,7 @@ TEST(VideoStreamAdapterTest,
|
|||
FakeVideoStream fake_stream(&adapter, 1280 * 720, 30,
|
||||
kDefaultMinPixelsPerFrame);
|
||||
// Apply adaptation but don't update the input.
|
||||
adapter.ApplyAdaptation(adapter.GetAdaptationDown());
|
||||
adapter.ApplyAdaptation(adapter.GetAdaptationDown(), nullptr);
|
||||
EXPECT_EQ(Adaptation::Status::kAwaitingPreviousAdaptation,
|
||||
adapter.GetAdaptationDown().status());
|
||||
adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION);
|
||||
|
|
@ -676,6 +698,36 @@ TEST(VideoStreamAdapterTest,
|
|||
EXPECT_EQ(Adaptation::Status::kValid, adaptation.status());
|
||||
}
|
||||
|
||||
TEST(VideoStreamAdapterTest, RestrictionsBroadcasted) {
|
||||
VideoStreamAdapter adapter;
|
||||
FakeVideoStreamAdapterListner listener;
|
||||
adapter.AddRestrictionsListener(&listener);
|
||||
adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE);
|
||||
FakeVideoStream fake_stream(&adapter, 1280 * 720, 30,
|
||||
kDefaultMinPixelsPerFrame);
|
||||
// Not broadcast on invalid ApplyAdaptation.
|
||||
{
|
||||
Adaptation adaptation = adapter.GetAdaptationUp();
|
||||
adapter.ApplyAdaptation(adaptation, nullptr);
|
||||
EXPECT_EQ(0, listener.calls());
|
||||
}
|
||||
|
||||
// Broadcast on ApplyAdaptation.
|
||||
{
|
||||
Adaptation adaptation = adapter.GetAdaptationDown();
|
||||
VideoStreamAdapter::RestrictionsWithCounters peek =
|
||||
adapter.PeekNextRestrictions(adaptation);
|
||||
fake_stream.ApplyAdaptation(adaptation);
|
||||
EXPECT_EQ(1, listener.calls());
|
||||
EXPECT_EQ(peek.restrictions, listener.last_restrictions());
|
||||
}
|
||||
|
||||
// Broadcast on ClearRestrictions().
|
||||
adapter.ClearRestrictions();
|
||||
EXPECT_EQ(2, listener.calls());
|
||||
EXPECT_EQ(VideoSourceRestrictions(), listener.last_restrictions());
|
||||
}
|
||||
|
||||
TEST(VideoStreamAdapterTest, PeekNextRestrictions) {
|
||||
VideoStreamAdapter adapter;
|
||||
// Any non-disabled DegradationPreference will do.
|
||||
|
|
@ -718,12 +770,24 @@ TEST(VideoStreamAdapterTest, PeekNextRestrictions) {
|
|||
}
|
||||
}
|
||||
|
||||
TEST(VideoStreamAdapterTest, PeekRestrictionsDoesNotBroadcast) {
|
||||
VideoStreamAdapter adapter;
|
||||
FakeVideoStreamAdapterListner listener;
|
||||
adapter.AddRestrictionsListener(&listener);
|
||||
adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE);
|
||||
FakeVideoStream fake_stream(&adapter, 1280 * 720, 30,
|
||||
kDefaultMinPixelsPerFrame);
|
||||
Adaptation adaptation = adapter.GetAdaptationDown();
|
||||
adapter.PeekNextRestrictions(adaptation);
|
||||
EXPECT_EQ(0, listener.calls());
|
||||
}
|
||||
|
||||
TEST(VideoStreamAdapterTest,
|
||||
SetDegradationPreferenceToOrFromBalancedClearsRestrictions) {
|
||||
VideoStreamAdapter adapter;
|
||||
adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE);
|
||||
adapter.SetInput(InputState(1280 * 720, 30, kDefaultMinPixelsPerFrame));
|
||||
adapter.ApplyAdaptation(adapter.GetAdaptationDown());
|
||||
adapter.ApplyAdaptation(adapter.GetAdaptationDown(), nullptr);
|
||||
EXPECT_NE(VideoSourceRestrictions(), adapter.source_restrictions());
|
||||
EXPECT_NE(0, adapter.adaptation_counters().Total());
|
||||
// Changing from non-balanced to balanced clears the restrictions.
|
||||
|
|
@ -731,7 +795,7 @@ TEST(VideoStreamAdapterTest,
|
|||
EXPECT_EQ(VideoSourceRestrictions(), adapter.source_restrictions());
|
||||
EXPECT_EQ(0, adapter.adaptation_counters().Total());
|
||||
// Apply adaptation again.
|
||||
adapter.ApplyAdaptation(adapter.GetAdaptationDown());
|
||||
adapter.ApplyAdaptation(adapter.GetAdaptationDown(), nullptr);
|
||||
EXPECT_NE(VideoSourceRestrictions(), adapter.source_restrictions());
|
||||
EXPECT_NE(0, adapter.adaptation_counters().Total());
|
||||
// Changing from balanced to non-balanced clears the restrictions.
|
||||
|
|
@ -752,7 +816,7 @@ TEST(VideoStreamAdapterDeathTest,
|
|||
adapter.SetInput(InputState(1280 * 720, 30, kDefaultMinPixelsPerFrame));
|
||||
Adaptation adaptation = adapter.GetAdaptationDown();
|
||||
adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION);
|
||||
EXPECT_DEATH(adapter.ApplyAdaptation(adaptation), "");
|
||||
EXPECT_DEATH(adapter.ApplyAdaptation(adaptation, nullptr), "");
|
||||
}
|
||||
|
||||
TEST(VideoStreamAdapterDeathTest, SetInputInvalidatesAdaptations) {
|
||||
|
|
|
|||
|
|
@ -316,11 +316,13 @@ void VideoStreamEncoderResourceManager::Initialize(
|
|||
}
|
||||
|
||||
void VideoStreamEncoderResourceManager::SetAdaptationProcessor(
|
||||
ResourceAdaptationProcessorInterface* adaptation_processor) {
|
||||
ResourceAdaptationProcessorInterface* adaptation_processor,
|
||||
VideoStreamAdapter* stream_adapter) {
|
||||
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
|
||||
adaptation_processor_ = adaptation_processor;
|
||||
balanced_constraint_->SetAdaptationProcessor(adaptation_processor);
|
||||
quality_scaler_resource_->SetAdaptationProcessor(adaptation_processor);
|
||||
stream_adapter_ = stream_adapter;
|
||||
}
|
||||
|
||||
void VideoStreamEncoderResourceManager::SetDegradationPreferences(
|
||||
|
|
@ -596,7 +598,8 @@ int VideoStreamEncoderResourceManager::LastInputFrameSizeOrDefault() const {
|
|||
void VideoStreamEncoderResourceManager::OnVideoSourceRestrictionsUpdated(
|
||||
VideoSourceRestrictions restrictions,
|
||||
const VideoAdaptationCounters& adaptation_counters,
|
||||
rtc::scoped_refptr<Resource> reason) {
|
||||
rtc::scoped_refptr<Resource> reason,
|
||||
const VideoSourceRestrictions& unfiltered_restrictions) {
|
||||
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
|
||||
// TODO(bugs.webrtc.org/11553) Remove reason parameter and add reset callback.
|
||||
if (!reason && adaptation_counters.Total() == 0) {
|
||||
|
|
@ -608,7 +611,8 @@ void VideoStreamEncoderResourceManager::OnVideoSourceRestrictionsUpdated(
|
|||
// means that if the task gets executed, |this| has not been freed yet.
|
||||
encoder_queue_->PostTask([this, restrictions] {
|
||||
RTC_DCHECK_RUN_ON(encoder_queue_);
|
||||
video_source_restrictions_ = restrictions;
|
||||
video_source_restrictions_ = FilterRestrictionsByDegradationPreference(
|
||||
restrictions, degradation_preference_);
|
||||
MaybeUpdateTargetFrameRate();
|
||||
});
|
||||
}
|
||||
|
|
@ -720,13 +724,12 @@ void VideoStreamEncoderResourceManager::OnQualityRampUp() {
|
|||
// the adaptation queue, add logic to prevent use-after-free on |this|.
|
||||
resource_adaptation_queue_->PostTask([this] {
|
||||
RTC_DCHECK_RUN_ON(resource_adaptation_queue_);
|
||||
if (!adaptation_processor_) {
|
||||
if (!stream_adapter_) {
|
||||
// The processor nulled before this task had a chance to execute. This
|
||||
// happens if the processor is destroyed. No action needed.
|
||||
return;
|
||||
}
|
||||
RTC_LOG(LS_INFO) << "Reset quality limitations.";
|
||||
adaptation_processor_->ResetVideoSourceRestrictions();
|
||||
stream_adapter_->ClearRestrictions();
|
||||
});
|
||||
quality_rampup_experiment_.reset();
|
||||
}
|
||||
|
|
|
|||
|
|
@ -65,6 +65,7 @@ extern const int kDefaultInputPixelsHeight;
|
|||
// ResourceAdaptationProcessor code such as the inital frame dropping.
|
||||
class VideoStreamEncoderResourceManager
|
||||
: public VideoSourceRestrictionsListener,
|
||||
public ResourceLimitationsListener,
|
||||
public QualityRampUpExperimentListener {
|
||||
public:
|
||||
VideoStreamEncoderResourceManager(
|
||||
|
|
@ -78,7 +79,8 @@ class VideoStreamEncoderResourceManager
|
|||
void Initialize(rtc::TaskQueue* encoder_queue,
|
||||
rtc::TaskQueue* resource_adaptation_queue);
|
||||
void SetAdaptationProcessor(
|
||||
ResourceAdaptationProcessorInterface* adaptation_processor);
|
||||
ResourceAdaptationProcessorInterface* adaptation_processor,
|
||||
VideoStreamAdapter* stream_adapter);
|
||||
|
||||
// TODO(https://crbug.com/webrtc/11563): The degradation preference is a
|
||||
// setting of the Processor, it does not belong to the Manager - can we get
|
||||
|
|
@ -130,7 +132,8 @@ class VideoStreamEncoderResourceManager
|
|||
void OnVideoSourceRestrictionsUpdated(
|
||||
VideoSourceRestrictions restrictions,
|
||||
const VideoAdaptationCounters& adaptation_counters,
|
||||
rtc::scoped_refptr<Resource> reason) override;
|
||||
rtc::scoped_refptr<Resource> reason,
|
||||
const VideoSourceRestrictions& unfiltered_restrictions) override;
|
||||
void OnResourceLimitationChanged(
|
||||
rtc::scoped_refptr<Resource> resource,
|
||||
const std::map<rtc::scoped_refptr<Resource>, VideoAdaptationCounters>&
|
||||
|
|
@ -238,6 +241,8 @@ class VideoStreamEncoderResourceManager
|
|||
RTC_GUARDED_BY(encoder_queue_);
|
||||
ResourceAdaptationProcessorInterface* adaptation_processor_
|
||||
RTC_GUARDED_BY(resource_adaptation_queue_);
|
||||
VideoStreamAdapter* stream_adapter_
|
||||
RTC_GUARDED_BY(resource_adaptation_queue_);
|
||||
// Thread-safe.
|
||||
VideoStreamEncoderObserver* const encoder_stats_observer_;
|
||||
|
||||
|
|
|
|||
|
|
@ -256,10 +256,12 @@ VideoStreamEncoder::VideoStreamEncoder(
|
|||
ParseAutomatincAnimationDetectionFieldTrial()),
|
||||
encoder_switch_requested_(false),
|
||||
input_state_provider_(encoder_stats_observer),
|
||||
video_stream_adapter_(std::make_unique<VideoStreamAdapter>()),
|
||||
resource_adaptation_processor_(
|
||||
std::make_unique<ResourceAdaptationProcessor>(
|
||||
&input_state_provider_,
|
||||
encoder_stats_observer)),
|
||||
encoder_stats_observer,
|
||||
video_stream_adapter_.get())),
|
||||
adaptation_constraints_(),
|
||||
adaptation_listeners_(),
|
||||
stream_resource_manager_(&input_state_provider_,
|
||||
|
|
@ -287,10 +289,11 @@ VideoStreamEncoder::VideoStreamEncoder(
|
|||
resource_adaptation_processor_->SetResourceAdaptationQueue(
|
||||
resource_adaptation_queue_.Get());
|
||||
stream_resource_manager_.SetAdaptationProcessor(
|
||||
resource_adaptation_processor_.get());
|
||||
resource_adaptation_processor_->AddRestrictionsListener(
|
||||
resource_adaptation_processor_.get(), video_stream_adapter_.get());
|
||||
resource_adaptation_processor_->AddResourceLimitationsListener(
|
||||
&stream_resource_manager_);
|
||||
resource_adaptation_processor_->AddRestrictionsListener(this);
|
||||
video_stream_adapter_->AddRestrictionsListener(&stream_resource_manager_);
|
||||
video_stream_adapter_->AddRestrictionsListener(this);
|
||||
|
||||
// Add the stream resource manager's resources to the processor.
|
||||
adaptation_constraints_ = stream_resource_manager_.AdaptationConstraints();
|
||||
|
|
@ -333,10 +336,12 @@ void VideoStreamEncoder::Stop() {
|
|||
for (auto* listener : adaptation_listeners_) {
|
||||
resource_adaptation_processor_->RemoveAdaptationListener(listener);
|
||||
}
|
||||
resource_adaptation_processor_->RemoveRestrictionsListener(this);
|
||||
resource_adaptation_processor_->RemoveRestrictionsListener(
|
||||
video_stream_adapter_->RemoveRestrictionsListener(this);
|
||||
video_stream_adapter_->RemoveRestrictionsListener(
|
||||
&stream_resource_manager_);
|
||||
stream_resource_manager_.SetAdaptationProcessor(nullptr);
|
||||
resource_adaptation_processor_->RemoveResourceLimitationsListener(
|
||||
&stream_resource_manager_);
|
||||
stream_resource_manager_.SetAdaptationProcessor(nullptr, nullptr);
|
||||
resource_adaptation_processor_.reset();
|
||||
}
|
||||
shutdown_adaptation_processor_event.Set();
|
||||
|
|
@ -1759,7 +1764,8 @@ bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const {
|
|||
void VideoStreamEncoder::OnVideoSourceRestrictionsUpdated(
|
||||
VideoSourceRestrictions restrictions,
|
||||
const VideoAdaptationCounters& adaptation_counters,
|
||||
rtc::scoped_refptr<Resource> reason) {
|
||||
rtc::scoped_refptr<Resource> reason,
|
||||
const VideoSourceRestrictions& unfiltered_restrictions) {
|
||||
RTC_DCHECK_RUN_ON(&resource_adaptation_queue_);
|
||||
std::string resource_name = reason ? reason->Name() : "<null>";
|
||||
RTC_LOG(INFO) << "Updating sink restrictions from " << resource_name << " to "
|
||||
|
|
@ -2102,8 +2108,7 @@ void VideoStreamEncoder::AddRestrictionsListenerForTesting(
|
|||
resource_adaptation_queue_.PostTask([this, restrictions_listener, &event] {
|
||||
RTC_DCHECK_RUN_ON(&resource_adaptation_queue_);
|
||||
RTC_DCHECK(resource_adaptation_processor_);
|
||||
resource_adaptation_processor_->AddRestrictionsListener(
|
||||
restrictions_listener);
|
||||
video_stream_adapter_->AddRestrictionsListener(restrictions_listener);
|
||||
event.Set();
|
||||
});
|
||||
event.Wait(rtc::Event::kForever);
|
||||
|
|
@ -2115,8 +2120,7 @@ void VideoStreamEncoder::RemoveRestrictionsListenerForTesting(
|
|||
resource_adaptation_queue_.PostTask([this, restrictions_listener, &event] {
|
||||
RTC_DCHECK_RUN_ON(&resource_adaptation_queue_);
|
||||
RTC_DCHECK(resource_adaptation_processor_);
|
||||
resource_adaptation_processor_->RemoveRestrictionsListener(
|
||||
restrictions_listener);
|
||||
video_stream_adapter_->RemoveRestrictionsListener(restrictions_listener);
|
||||
event.Set();
|
||||
});
|
||||
event.Wait(rtc::Event::kForever);
|
||||
|
|
|
|||
|
|
@ -120,7 +120,8 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
|
|||
void OnVideoSourceRestrictionsUpdated(
|
||||
VideoSourceRestrictions restrictions,
|
||||
const VideoAdaptationCounters& adaptation_counters,
|
||||
rtc::scoped_refptr<Resource> reason) override;
|
||||
rtc::scoped_refptr<Resource> reason,
|
||||
const VideoSourceRestrictions& unfiltered_restrictions) override;
|
||||
|
||||
// Used for injected test resources.
|
||||
// TODO(eshr): Move all adaptation tests out of VideoStreamEncoder tests.
|
||||
|
|
@ -408,6 +409,9 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface,
|
|||
// Provies video stream input states: current resolution and frame rate.
|
||||
// This class is thread-safe.
|
||||
VideoStreamInputStateProvider input_state_provider_;
|
||||
|
||||
std::unique_ptr<VideoStreamAdapter> video_stream_adapter_
|
||||
RTC_GUARDED_BY(&resource_adaptation_queue_);
|
||||
// Responsible for adapting input resolution or frame rate to ensure resources
|
||||
// (e.g. CPU or bandwidth) are not overused.
|
||||
// This class is single-threaded on the resource adaptation queue.
|
||||
|
|
|
|||
|
|
@ -205,7 +205,8 @@ class FakeVideoSourceRestrictionsListener
|
|||
void OnVideoSourceRestrictionsUpdated(
|
||||
VideoSourceRestrictions restrictions,
|
||||
const VideoAdaptationCounters& adaptation_counters,
|
||||
rtc::scoped_refptr<Resource> reason) override {
|
||||
rtc::scoped_refptr<Resource> reason,
|
||||
const VideoSourceRestrictions& unfiltered_restrictions) override {
|
||||
was_restrictions_updated_ = true;
|
||||
restrictions_updated_event_.Set();
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue