diff --git a/call/adaptation/resource_adaptation_processor.cc b/call/adaptation/resource_adaptation_processor.cc index 0be01cb794..3c93781e14 100644 --- a/call/adaptation/resource_adaptation_processor.cc +++ b/call/adaptation/resource_adaptation_processor.cc @@ -15,6 +15,8 @@ #include #include "absl/algorithm/container.h" +#include "api/video/video_adaptation_counters.h" +#include "call/adaptation/video_stream_adapter.h" #include "rtc_base/logging.h" #include "rtc_base/ref_counted_object.h" #include "rtc_base/strings/string_builder.h" @@ -67,7 +69,8 @@ ResourceAdaptationProcessor::MitigationResultAndLogMessage:: ResourceAdaptationProcessor::ResourceAdaptationProcessor( VideoStreamInputStateProvider* input_state_provider, - VideoStreamEncoderObserver* encoder_stats_observer) + VideoStreamEncoderObserver* encoder_stats_observer, + VideoStreamAdapter* stream_adapter) : resource_adaptation_queue_(nullptr), resource_listener_delegate_( new rtc::RefCountedObject(this)), @@ -77,16 +80,15 @@ ResourceAdaptationProcessor::ResourceAdaptationProcessor( degradation_preference_(DegradationPreference::DISABLED), effective_degradation_preference_(DegradationPreference::DISABLED), is_screenshare_(false), - stream_adapter_(std::make_unique()), + stream_adapter_(stream_adapter), last_reported_source_restrictions_(), previous_mitigation_results_(), - processing_in_progress_(false) {} + processing_in_progress_(false) { + RTC_DCHECK(stream_adapter_); +} ResourceAdaptationProcessor::~ResourceAdaptationProcessor() { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); - RTC_DCHECK(restrictions_listeners_.empty()) - << "There are restrictions listener(s) depending on a " - << "ResourceAdaptationProcessor being destroyed."; RTC_DCHECK(resources_.empty()) << "There are resource(s) attached to a ResourceAdaptationProcessor " << "being destroyed."; @@ -96,6 +98,7 @@ ResourceAdaptationProcessor::~ResourceAdaptationProcessor() { RTC_DCHECK(adaptation_listeners_.empty()) << "There are listener(s) attached to a ResourceAdaptationProcessor " << "being destroyed."; + stream_adapter_->RemoveRestrictionsListener(this); resource_listener_delegate_->OnProcessorDestroyed(); } @@ -107,6 +110,8 @@ void ResourceAdaptationProcessor::SetResourceAdaptationQueue( resource_listener_delegate_->SetResourceAdaptationQueue( resource_adaptation_queue); RTC_DCHECK_RUN_ON(resource_adaptation_queue_); + // Now that we have the adaptation queue we can attach as adaptation listener. + stream_adapter_->AddRestrictionsListener(this); } DegradationPreference ResourceAdaptationProcessor::degradation_preference() @@ -121,22 +126,23 @@ ResourceAdaptationProcessor::effective_degradation_preference() const { return effective_degradation_preference_; } -void ResourceAdaptationProcessor::AddRestrictionsListener( - VideoSourceRestrictionsListener* restrictions_listener) { +void ResourceAdaptationProcessor::AddResourceLimitationsListener( + ResourceLimitationsListener* limitations_listener) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); - RTC_DCHECK(std::find(restrictions_listeners_.begin(), - restrictions_listeners_.end(), - restrictions_listener) == restrictions_listeners_.end()); - restrictions_listeners_.push_back(restrictions_listener); + RTC_DCHECK(std::find(resource_limitations_listeners_.begin(), + resource_limitations_listeners_.end(), + limitations_listener) == + resource_limitations_listeners_.end()); + resource_limitations_listeners_.push_back(limitations_listener); } - -void ResourceAdaptationProcessor::RemoveRestrictionsListener( - VideoSourceRestrictionsListener* restrictions_listener) { +void ResourceAdaptationProcessor::RemoveResourceLimitationsListener( + ResourceLimitationsListener* limitations_listener) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); - auto it = std::find(restrictions_listeners_.begin(), - restrictions_listeners_.end(), restrictions_listener); - RTC_DCHECK(it != restrictions_listeners_.end()); - restrictions_listeners_.erase(it); + auto it = + std::find(resource_limitations_listeners_.begin(), + resource_limitations_listeners_.end(), limitations_listener); + RTC_DCHECK(it != resource_limitations_listeners_.end()); + resource_limitations_listeners_.erase(it); } void ResourceAdaptationProcessor::AddResource( @@ -232,39 +238,6 @@ void ResourceAdaptationProcessor::MaybeUpdateEffectiveDegradationPreference() { ? DegradationPreference::MAINTAIN_RESOLUTION : degradation_preference_; stream_adapter_->SetDegradationPreference(effective_degradation_preference_); - MaybeUpdateVideoSourceRestrictions(nullptr); -} - -void ResourceAdaptationProcessor::ResetVideoSourceRestrictions() { - RTC_DCHECK_RUN_ON(resource_adaptation_queue_); - RTC_LOG(INFO) << "Resetting restrictions"; - stream_adapter_->ClearRestrictions(); - adaptation_limits_by_resources_.clear(); - for (auto restrictions_listener : restrictions_listeners_) { - restrictions_listener->OnResourceLimitationChanged(nullptr, {}); - } - MaybeUpdateVideoSourceRestrictions(nullptr); -} - -void ResourceAdaptationProcessor::MaybeUpdateVideoSourceRestrictions( - rtc::scoped_refptr reason) { - RTC_DCHECK_RUN_ON(resource_adaptation_queue_); - VideoSourceRestrictions new_source_restrictions = - FilterRestrictionsByDegradationPreference( - stream_adapter_->source_restrictions(), - effective_degradation_preference_); - if (last_reported_source_restrictions_ != new_source_restrictions) { - RTC_LOG(INFO) << "Reporting new restrictions (in " - << DegradationPreferenceToString( - effective_degradation_preference_) - << "): " << new_source_restrictions.ToString(); - last_reported_source_restrictions_ = std::move(new_source_restrictions); - for (auto* restrictions_listener : restrictions_listeners_) { - restrictions_listener->OnVideoSourceRestrictionsUpdated( - last_reported_source_restrictions_, - stream_adapter_->adaptation_counters(), reason); - } - } } void ResourceAdaptationProcessor::OnResourceUsageStateMeasured( @@ -386,10 +359,10 @@ ResourceAdaptationProcessor::OnResourceUnderuse( MitigationResult::kNotMostLimitedResource, message.Release()); } - UpdateResourceLimitations(reason_resource, peek_restrictions); if (most_limited_resources.size() > 1) { // If there are multiple most limited resources, all must signal underuse // before the adaptation is applied. + UpdateResourceLimitations(reason_resource, peek_restrictions); processing_in_progress_ = false; rtc::StringBuilder message; message << "Resource \"" << reason_resource->Name() @@ -399,14 +372,11 @@ ResourceAdaptationProcessor::OnResourceUnderuse( } } // Apply adaptation. - stream_adapter_->ApplyAdaptation(adaptation); + stream_adapter_->ApplyAdaptation(adaptation, reason_resource); for (auto* adaptation_listener : adaptation_listeners_) { adaptation_listener->OnAdaptationApplied( input_state, restrictions_before, restrictions_after, reason_resource); } - // Update VideoSourceRestrictions based on adaptation. This also informs the - // |restrictions_listeners_|. - MaybeUpdateVideoSourceRestrictions(reason_resource); processing_in_progress_ = false; rtc::StringBuilder message; message << "Adapted up successfully. Unfiltered adaptations: " @@ -456,15 +426,11 @@ ResourceAdaptationProcessor::OnResourceOveruse( stream_adapter_->PeekNextRestrictions(adaptation); VideoSourceRestrictions restrictions_after = peek_next_restrictions.restrictions; - UpdateResourceLimitations(reason_resource, peek_next_restrictions); - stream_adapter_->ApplyAdaptation(adaptation); + stream_adapter_->ApplyAdaptation(adaptation, reason_resource); for (auto* adaptation_listener : adaptation_listeners_) { adaptation_listener->OnAdaptationApplied( input_state, restrictions_before, restrictions_after, reason_resource); } - // Update VideoSourceRestrictions based on adaptation. This also informs the - // |restrictions_listeners_|. - MaybeUpdateVideoSourceRestrictions(reason_resource); processing_in_progress_ = false; rtc::StringBuilder message; message << "Adapted down successfully. Unfiltered adaptations: " @@ -520,18 +486,20 @@ ResourceAdaptationProcessor::FindMostLimitedResources() const { void ResourceAdaptationProcessor::UpdateResourceLimitations( rtc::scoped_refptr reason_resource, - const VideoStreamAdapter::RestrictionsWithCounters& - peek_next_restrictions) { - adaptation_limits_by_resources_[reason_resource] = peek_next_restrictions; + const VideoStreamAdapter::RestrictionsWithCounters& restrictions) { + auto& adaptation_limits = adaptation_limits_by_resources_[reason_resource]; + if (adaptation_limits == restrictions) { + return; + } + adaptation_limits = restrictions; std::map, VideoAdaptationCounters> limitations; for (const auto& p : adaptation_limits_by_resources_) { limitations.insert(std::make_pair(p.first, p.second.adaptation_counters)); } - - for (auto restrictions_listener : restrictions_listeners_) { - restrictions_listener->OnResourceLimitationChanged(reason_resource, - limitations); + for (auto limitations_listener : resource_limitations_listeners_) { + limitations_listener->OnResourceLimitationChanged(reason_resource, + limitations); } } @@ -539,8 +507,8 @@ void ResourceAdaptationProcessor:: MaybeUpdateResourceLimitationsOnResourceRemoval( VideoStreamAdapter::RestrictionsWithCounters removed_limitations) { if (adaptation_limits_by_resources_.empty()) { - // Only the resource being removed was adapted so reset restrictions. - ResetVideoSourceRestrictions(); + // Only the resource being removed was adapted so clear restrictions. + stream_adapter_->ClearRestrictions(); return; } @@ -558,20 +526,37 @@ void ResourceAdaptationProcessor:: Adaptation adapt_to = stream_adapter_->GetAdaptationTo( most_limited.adaptation_counters, most_limited.restrictions); RTC_DCHECK_EQ(adapt_to.status(), Adaptation::Status::kValid); - stream_adapter_->ApplyAdaptation(adapt_to); - - RTC_LOG(INFO) << "Most limited resource removed. Restoring restrictions to " - "next most limited restrictions: " - << most_limited.restrictions.ToString() << " with counters " - << most_limited.adaptation_counters.ToString(); - - MaybeUpdateVideoSourceRestrictions(nullptr); + stream_adapter_->ApplyAdaptation(adapt_to, nullptr); auto input_state = input_state_provider_->InputState(); for (auto* adaptation_listener : adaptation_listeners_) { adaptation_listener->OnAdaptationApplied( input_state, removed_limitations.restrictions, most_limited.restrictions, nullptr); } + + RTC_LOG(INFO) << "Most limited resource removed. Restoring restrictions to " + "next most limited restrictions: " + << most_limited.restrictions.ToString() << " with counters " + << most_limited.adaptation_counters.ToString(); +} + +void ResourceAdaptationProcessor::OnVideoSourceRestrictionsUpdated( + VideoSourceRestrictions restrictions, + const VideoAdaptationCounters& adaptation_counters, + rtc::scoped_refptr reason, + const VideoSourceRestrictions& unfiltered_restrictions) { + RTC_DCHECK_RUN_ON(resource_adaptation_queue_); + if (reason) { + UpdateResourceLimitations(reason, + {unfiltered_restrictions, adaptation_counters}); + } else if (adaptation_counters.Total() == 0) { + // Adaptations are cleared. + adaptation_limits_by_resources_.clear(); + previous_mitigation_results_.clear(); + for (auto limitations_listener : resource_limitations_listeners_) { + limitations_listener->OnResourceLimitationChanged(nullptr, {}); + } + } } } // namespace webrtc diff --git a/call/adaptation/resource_adaptation_processor.h b/call/adaptation/resource_adaptation_processor.h index cff50955e7..f9dec8946e 100644 --- a/call/adaptation/resource_adaptation_processor.h +++ b/call/adaptation/resource_adaptation_processor.h @@ -51,11 +51,13 @@ namespace webrtc { // any thread but MUST subsequently be used and destroyed on a single sequence, // i.e. the "resource adaptation task queue". class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, + public VideoSourceRestrictionsListener, public ResourceListener { public: ResourceAdaptationProcessor( VideoStreamInputStateProvider* input_state_provider, - VideoStreamEncoderObserver* encoder_stats_observer); + VideoStreamEncoderObserver* encoder_stats_observer, + VideoStreamAdapter* video_stream_adapter); ~ResourceAdaptationProcessor() override; void SetResourceAdaptationQueue( @@ -65,10 +67,10 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, DegradationPreference degradation_preference() const override; DegradationPreference effective_degradation_preference() const override; - void AddRestrictionsListener( - VideoSourceRestrictionsListener* restrictions_listener) override; - void RemoveRestrictionsListener( - VideoSourceRestrictionsListener* restrictions_listener) override; + void AddResourceLimitationsListener( + ResourceLimitationsListener* limitations_listener) override; + void RemoveResourceLimitationsListener( + ResourceLimitationsListener* limitations_listener) override; void AddResource(rtc::scoped_refptr resource) override; std::vector> GetResources() const override; void RemoveResource(rtc::scoped_refptr resource) override; @@ -83,13 +85,19 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, void SetDegradationPreference( DegradationPreference degradation_preference) override; void SetIsScreenshare(bool is_screenshare) override; - void ResetVideoSourceRestrictions() override; // ResourceListener implementation. // Triggers OnResourceUnderuse() or OnResourceOveruse(). void OnResourceUsageStateMeasured(rtc::scoped_refptr resource, ResourceUsageState usage_state) override; + // VideoSourceRestrictionsListener implementation. + void OnVideoSourceRestrictionsUpdated( + VideoSourceRestrictions restrictions, + const VideoAdaptationCounters& adaptation_counters, + rtc::scoped_refptr reason, + const VideoSourceRestrictions& unfiltered_restrictions) override; + // May trigger 1-2 adaptations. It is meant to reduce resolution but this is // not guaranteed. It may adapt frame rate, which does not address the issue. // TODO(hbos): Can we get rid of this? @@ -149,9 +157,6 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, // Needs to be invoked any time |degradation_preference_| or |is_screenshare_| // changes to ensure |effective_degradation_preference_| is up-to-date. void MaybeUpdateEffectiveDegradationPreference(); - // If the filtered source restrictions are different than - // |last_reported_source_restrictions_|, inform the listeners. - void MaybeUpdateVideoSourceRestrictions(rtc::scoped_refptr reason); void UpdateResourceLimitations( rtc::scoped_refptr reason_resource, @@ -178,7 +183,7 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, RTC_GUARDED_BY(resource_adaptation_queue_); VideoStreamEncoderObserver* const encoder_stats_observer_ RTC_GUARDED_BY(resource_adaptation_queue_); - std::vector restrictions_listeners_ + std::vector resource_limitations_listeners_ RTC_GUARDED_BY(resource_adaptation_queue_); std::vector> resources_ RTC_GUARDED_BY(resource_adaptation_queue_); @@ -198,7 +203,7 @@ class ResourceAdaptationProcessor : public ResourceAdaptationProcessorInterface, RTC_GUARDED_BY(resource_adaptation_queue_); bool is_screenshare_ RTC_GUARDED_BY(resource_adaptation_queue_); // Responsible for generating and applying possible adaptations. - const std::unique_ptr stream_adapter_ + VideoStreamAdapter* const stream_adapter_ RTC_GUARDED_BY(resource_adaptation_queue_); VideoSourceRestrictions last_reported_source_restrictions_ RTC_GUARDED_BY(resource_adaptation_queue_); diff --git a/call/adaptation/resource_adaptation_processor_interface.cc b/call/adaptation/resource_adaptation_processor_interface.cc index 48ddf65ed3..79f099b267 100644 --- a/call/adaptation/resource_adaptation_processor_interface.cc +++ b/call/adaptation/resource_adaptation_processor_interface.cc @@ -12,9 +12,9 @@ namespace webrtc { -VideoSourceRestrictionsListener::~VideoSourceRestrictionsListener() = default; - ResourceAdaptationProcessorInterface::~ResourceAdaptationProcessorInterface() = default; +ResourceLimitationsListener::~ResourceLimitationsListener() = default; + } // namespace webrtc diff --git a/call/adaptation/resource_adaptation_processor_interface.h b/call/adaptation/resource_adaptation_processor_interface.h index a97fe8efe4..bf6b19790b 100644 --- a/call/adaptation/resource_adaptation_processor_interface.h +++ b/call/adaptation/resource_adaptation_processor_interface.h @@ -28,26 +28,16 @@ namespace webrtc { -// The listener is responsible for carrying out the reconfiguration of the video -// source such that the VideoSourceRestrictions are fulfilled. -class VideoSourceRestrictionsListener { +class ResourceLimitationsListener { public: - virtual ~VideoSourceRestrictionsListener(); - - // The |restrictions| are filtered by degradation preference but not the - // |adaptation_counters|, which are currently only reported for legacy stats - // calculation purposes. - virtual void OnVideoSourceRestrictionsUpdated( - VideoSourceRestrictions restrictions, - const VideoAdaptationCounters& adaptation_counters, - rtc::scoped_refptr reason) = 0; + virtual ~ResourceLimitationsListener(); // The limitations on a resource were changed. This does not mean the current // video restrictions have changed. virtual void OnResourceLimitationChanged( rtc::scoped_refptr resource, const std::map, VideoAdaptationCounters>& - resource_limitations) {} + resource_limitations) = 0; }; // The Resource Adaptation Processor is responsible for reacting to resource @@ -74,10 +64,10 @@ class ResourceAdaptationProcessorInterface { // with AddResource() and RemoveResource() instead. When the processor is // multi-stream aware, stream-specific resouces will get added and removed // over time. - virtual void AddRestrictionsListener( - VideoSourceRestrictionsListener* restrictions_listener) = 0; - virtual void RemoveRestrictionsListener( - VideoSourceRestrictionsListener* restrictions_listener) = 0; + virtual void AddResourceLimitationsListener( + ResourceLimitationsListener* limitations_listener) = 0; + virtual void RemoveResourceLimitationsListener( + ResourceLimitationsListener* limitations_listener) = 0; virtual void AddResource(rtc::scoped_refptr resource) = 0; virtual std::vector> GetResources() const = 0; virtual void RemoveResource(rtc::scoped_refptr resource) = 0; @@ -93,7 +83,6 @@ class ResourceAdaptationProcessorInterface { virtual void SetDegradationPreference( DegradationPreference degradation_preference) = 0; virtual void SetIsScreenshare(bool is_screenshare) = 0; - virtual void ResetVideoSourceRestrictions() = 0; // May trigger one or more adaptations. It is meant to reduce resolution - // useful if a frame was dropped due to its size - however, the implementation diff --git a/call/adaptation/resource_adaptation_processor_unittest.cc b/call/adaptation/resource_adaptation_processor_unittest.cc index da1ab1cda1..b661333337 100644 --- a/call/adaptation/resource_adaptation_processor_unittest.cc +++ b/call/adaptation/resource_adaptation_processor_unittest.cc @@ -65,7 +65,8 @@ class VideoSourceRestrictionsListenerForTesting void OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions, const VideoAdaptationCounters& adaptation_counters, - rtc::scoped_refptr reason) override { + rtc::scoped_refptr reason, + const VideoSourceRestrictions& unfiltered_restrictions) override { RTC_DCHECK_RUN_ON(&sequence_checker_); ++restrictions_updated_count_; restrictions_ = restrictions; @@ -91,11 +92,13 @@ class ResourceAdaptationProcessorTest : public ::testing::Test { other_resource_(FakeResource::Create("OtherFakeResource")), adaptation_constraint_("FakeAdaptationConstraint"), adaptation_listener_(), + video_stream_adapter_(std::make_unique()), processor_(std::make_unique( &input_state_provider_, - /*encoder_stats_observer=*/&frame_rate_provider_)) { + /*encoder_stats_observer=*/&frame_rate_provider_, + video_stream_adapter_.get())) { processor_->SetResourceAdaptationQueue(TaskQueueBase::Current()); - processor_->AddRestrictionsListener(&restrictions_listener_); + video_stream_adapter_->AddRestrictionsListener(&restrictions_listener_); processor_->AddResource(resource_); processor_->AddResource(other_resource_); processor_->AddAdaptationConstraint(&adaptation_constraint_); @@ -122,7 +125,6 @@ class ResourceAdaptationProcessorTest : public ::testing::Test { } void DestroyProcessor() { - processor_->RemoveRestrictionsListener(&restrictions_listener_); if (resource_) { processor_->RemoveResource(resource_); } @@ -131,6 +133,7 @@ class ResourceAdaptationProcessorTest : public ::testing::Test { } processor_->RemoveAdaptationConstraint(&adaptation_constraint_); processor_->RemoveAdaptationListener(&adaptation_listener_); + video_stream_adapter_->RemoveRestrictionsListener(&restrictions_listener_); processor_.reset(); } @@ -145,6 +148,7 @@ class ResourceAdaptationProcessorTest : public ::testing::Test { rtc::scoped_refptr other_resource_; FakeAdaptationConstraint adaptation_constraint_; FakeAdaptationListener adaptation_listener_; + std::unique_ptr video_stream_adapter_; std::unique_ptr processor_; VideoSourceRestrictionsListenerForTesting restrictions_listener_; }; @@ -290,7 +294,7 @@ TEST_F(ResourceAdaptationProcessorTest, resource_->SetUsageState(ResourceUsageState::kOveruse); EXPECT_EQ(1u, restrictions_listener_.restrictions_updated_count()); - processor_->ResetVideoSourceRestrictions(); + video_stream_adapter_->ClearRestrictions(); EXPECT_EQ(0, restrictions_listener_.adaptation_counters().Total()); other_resource_->SetUsageState(ResourceUsageState::kOveruse); EXPECT_EQ(1, restrictions_listener_.adaptation_counters().Total()); diff --git a/call/adaptation/video_stream_adapter.cc b/call/adaptation/video_stream_adapter.cc index 4bf236fe71..bce27820d4 100644 --- a/call/adaptation/video_stream_adapter.cc +++ b/call/adaptation/video_stream_adapter.cc @@ -61,6 +61,8 @@ int GetLowerResolutionThan(int pixel_count) { } // namespace +VideoSourceRestrictionsListener::~VideoSourceRestrictionsListener() = default; + VideoSourceRestrictions FilterRestrictionsByDegradationPreference( VideoSourceRestrictions source_restrictions, DegradationPreference degradation_preference) { @@ -82,28 +84,6 @@ VideoSourceRestrictions FilterRestrictionsByDegradationPreference( return source_restrictions; } -VideoAdaptationCounters FilterVideoAdaptationCountersByDegradationPreference( - VideoAdaptationCounters counters, - DegradationPreference degradation_preference) { - switch (degradation_preference) { - case DegradationPreference::BALANCED: - break; - case DegradationPreference::MAINTAIN_FRAMERATE: - counters.fps_adaptations = 0; - break; - case DegradationPreference::MAINTAIN_RESOLUTION: - counters.resolution_adaptations = 0; - break; - case DegradationPreference::DISABLED: - counters.resolution_adaptations = 0; - counters.fps_adaptations = 0; - break; - default: - RTC_NOTREACHED(); - } - return counters; -} - // TODO(hbos): Use absl::optional<> instead? int GetHigherResolutionThan(int pixel_count) { return pixel_count != std::numeric_limits::max() @@ -355,40 +335,73 @@ VideoStreamAdapter::VideoStreamAdapter() adaptation_validation_id_(0), degradation_preference_(DegradationPreference::DISABLED), input_state_(), - last_adaptation_request_(absl::nullopt) {} + last_adaptation_request_(absl::nullopt), + last_video_source_restrictions_() { + sequence_checker_.Detach(); +} VideoStreamAdapter::~VideoStreamAdapter() {} VideoSourceRestrictions VideoStreamAdapter::source_restrictions() const { + RTC_DCHECK_RUN_ON(&sequence_checker_); return source_restrictor_->source_restrictions(); } const VideoAdaptationCounters& VideoStreamAdapter::adaptation_counters() const { + RTC_DCHECK_RUN_ON(&sequence_checker_); return source_restrictor_->adaptation_counters(); } void VideoStreamAdapter::ClearRestrictions() { + RTC_DCHECK_RUN_ON(&sequence_checker_); // Invalidate any previously returned Adaptation. + RTC_LOG(INFO) << "Resetting restrictions"; ++adaptation_validation_id_; source_restrictor_->ClearRestrictions(); last_adaptation_request_.reset(); + BroadcastVideoRestrictionsUpdate(nullptr); +} + +void VideoStreamAdapter::AddRestrictionsListener( + VideoSourceRestrictionsListener* restrictions_listener) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + RTC_DCHECK(std::find(restrictions_listeners_.begin(), + restrictions_listeners_.end(), + restrictions_listener) == restrictions_listeners_.end()); + restrictions_listeners_.push_back(restrictions_listener); +} + +void VideoStreamAdapter::RemoveRestrictionsListener( + VideoSourceRestrictionsListener* restrictions_listener) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + auto it = std::find(restrictions_listeners_.begin(), + restrictions_listeners_.end(), restrictions_listener); + RTC_DCHECK(it != restrictions_listeners_.end()); + restrictions_listeners_.erase(it); } void VideoStreamAdapter::SetDegradationPreference( DegradationPreference degradation_preference) { + RTC_DCHECK_RUN_ON(&sequence_checker_); if (degradation_preference_ == degradation_preference) return; // Invalidate any previously returned Adaptation. ++adaptation_validation_id_; - if (degradation_preference == DegradationPreference::BALANCED || - degradation_preference_ == DegradationPreference::BALANCED) { - ClearRestrictions(); - } + bool balanced_switch = + degradation_preference == DegradationPreference::BALANCED || + degradation_preference_ == DegradationPreference::BALANCED; degradation_preference_ = degradation_preference; + if (balanced_switch) { + // ClearRestrictions() calls BroadcastVideoRestrictionsUpdate(nullptr). + ClearRestrictions(); + } else { + BroadcastVideoRestrictionsUpdate(nullptr); + } } void VideoStreamAdapter::SetInput(VideoStreamInputState input_state) { // Invalidate any previously returned Adaptation. + RTC_DCHECK_RUN_ON(&sequence_checker_); ++adaptation_validation_id_; input_state_ = input_state; source_restrictor_->set_min_pixels_per_frame( @@ -396,6 +409,7 @@ void VideoStreamAdapter::SetInput(VideoStreamInputState input_state) { } Adaptation VideoStreamAdapter::GetAdaptationUp() const { + RTC_DCHECK_RUN_ON(&sequence_checker_); RTC_DCHECK_NE(degradation_preference_, DegradationPreference::DISABLED); RTC_DCHECK(input_state_.HasInputFrameSizeAndFramesPerSecond()); // Don't adapt if we're awaiting a previous adaptation to have an effect. @@ -469,6 +483,7 @@ Adaptation VideoStreamAdapter::GetAdaptationUp() const { } Adaptation VideoStreamAdapter::GetAdaptationDown() const { + RTC_DCHECK_RUN_ON(&sequence_checker_); RTC_DCHECK_NE(degradation_preference_, DegradationPreference::DISABLED); RTC_DCHECK(input_state_.HasInputFrameSizeAndFramesPerSecond()); // Don't adapt if we're awaiting a previous adaptation to have an effect or @@ -537,11 +552,13 @@ Adaptation VideoStreamAdapter::GetAdaptationDown() const { VideoStreamAdapter::RestrictionsWithCounters VideoStreamAdapter::PeekNextRestrictions(const Adaptation& adaptation) const { + RTC_DCHECK_RUN_ON(&sequence_checker_); RTC_DCHECK_EQ(adaptation.validation_id_, adaptation_validation_id_); RTC_LOG(LS_INFO) << "PeekNextRestrictions called"; - if (adaptation.status() != Adaptation::Status::kValid) + if (adaptation.status() != Adaptation::Status::kValid) { return {source_restrictor_->source_restrictions(), source_restrictor_->adaptation_counters()}; + } VideoSourceRestrictor restrictor_copy = *source_restrictor_; restrictor_copy.ApplyAdaptationStep(adaptation.step(), degradation_preference_); @@ -549,7 +566,10 @@ VideoStreamAdapter::PeekNextRestrictions(const Adaptation& adaptation) const { restrictor_copy.adaptation_counters()}; } -void VideoStreamAdapter::ApplyAdaptation(const Adaptation& adaptation) { +void VideoStreamAdapter::ApplyAdaptation( + const Adaptation& adaptation, + rtc::scoped_refptr resource) { + RTC_DCHECK_RUN_ON(&sequence_checker_); RTC_DCHECK_EQ(adaptation.validation_id_, adaptation_validation_id_); RTC_LOG(LS_INFO) << "ApplyAdaptation called"; if (adaptation.status() != Adaptation::Status::kValid) @@ -562,14 +582,33 @@ void VideoStreamAdapter::ApplyAdaptation(const Adaptation& adaptation) { // Adapt! source_restrictor_->ApplyAdaptationStep(adaptation.step(), degradation_preference_); + BroadcastVideoRestrictionsUpdate(resource); } Adaptation VideoStreamAdapter::GetAdaptationTo( const VideoAdaptationCounters& counters, const VideoSourceRestrictions& restrictions) const { // Adapts up/down from the current levels so counters are equal. + RTC_DCHECK_RUN_ON(&sequence_checker_); return Adaptation(adaptation_validation_id_, Adaptation::Step(restrictions, counters)); } +void VideoStreamAdapter::BroadcastVideoRestrictionsUpdate( + const rtc::scoped_refptr& resource) { + RTC_DCHECK_RUN_ON(&sequence_checker_); + VideoSourceRestrictions filtered = FilterRestrictionsByDegradationPreference( + source_restrictions(), degradation_preference_); + if (last_filtered_restrictions_ == filtered) { + return; + } + for (auto* restrictions_listener : restrictions_listeners_) { + restrictions_listener->OnVideoSourceRestrictionsUpdated( + filtered, source_restrictor_->adaptation_counters(), resource, + source_restrictions()); + } + last_video_source_restrictions_ = source_restrictor_->source_restrictions(); + last_filtered_restrictions_ = filtered; +} + } // namespace webrtc diff --git a/call/adaptation/video_stream_adapter.h b/call/adaptation/video_stream_adapter.h index 3a56f4f7c5..c82569d985 100644 --- a/call/adaptation/video_stream_adapter.h +++ b/call/adaptation/video_stream_adapter.h @@ -12,6 +12,7 @@ #define CALL_ADAPTATION_VIDEO_STREAM_ADAPTER_H_ #include +#include #include "absl/types/optional.h" #include "api/adaptation/resource.h" @@ -24,6 +25,22 @@ namespace webrtc { +// The listener is responsible for carrying out the reconfiguration of the video +// source such that the VideoSourceRestrictions are fulfilled. +class VideoSourceRestrictionsListener { + public: + virtual ~VideoSourceRestrictionsListener(); + + // The |restrictions| are filtered by degradation preference but not the + // |adaptation_counters|, which are currently only reported for legacy stats + // calculation purposes. + virtual void OnVideoSourceRestrictionsUpdated( + VideoSourceRestrictions restrictions, + const VideoAdaptationCounters& adaptation_counters, + rtc::scoped_refptr reason, + const VideoSourceRestrictions& unfiltered_restrictions) = 0; +}; + class VideoStreamAdapter; extern const int kMinFrameRateFps; @@ -32,10 +49,6 @@ VideoSourceRestrictions FilterRestrictionsByDegradationPreference( VideoSourceRestrictions source_restrictions, DegradationPreference degradation_preference); -VideoAdaptationCounters FilterVideoAdaptationCountersByDegradationPreference( - VideoAdaptationCounters counters, - DegradationPreference degradation_preference); - int GetHigherResolutionThan(int pixel_count); // Represents one step that the VideoStreamAdapter can take when adapting the @@ -128,6 +141,11 @@ class VideoStreamAdapter { const VideoAdaptationCounters& adaptation_counters() const; void ClearRestrictions(); + void AddRestrictionsListener( + VideoSourceRestrictionsListener* restrictions_listener); + void RemoveRestrictionsListener( + VideoSourceRestrictionsListener* restrictions_listener); + // TODO(hbos): Setting the degradation preference should not clear // restrictions! This is not defined in the spec and is unexpected, there is a // tiny risk that people would discover and rely on this behavior. @@ -145,6 +163,15 @@ class VideoStreamAdapter { struct RestrictionsWithCounters { VideoSourceRestrictions restrictions; VideoAdaptationCounters adaptation_counters; + + bool operator==(const RestrictionsWithCounters& other) { + return restrictions == other.restrictions && + adaptation_counters == other.adaptation_counters; + } + + bool operator!=(const RestrictionsWithCounters& other) { + return !(*this == other); + } }; // Returns the restrictions that result from applying the adaptation, without @@ -152,12 +179,19 @@ class VideoStreamAdapter { // are returned. RestrictionsWithCounters PeekNextRestrictions( const Adaptation& adaptation) const; - // Updates source_restrictions() based according to the Adaptation. - void ApplyAdaptation(const Adaptation& adaptation); + // Updates source_restrictions() based according to the Adaptation. These + // adaptations will be attributed to the Resource |resource| if the |resource| + // is non-null. If |resource| is null the adaptation will be changed in + // general, and thus could be adapted up in the future from other resources. + void ApplyAdaptation(const Adaptation& adaptation, + rtc::scoped_refptr resource); private: class VideoSourceRestrictor; + void BroadcastVideoRestrictionsUpdate( + const rtc::scoped_refptr& resource); + // The input frame rate and resolution at the time of an adaptation in the // direction described by |mode_| (up or down). // TODO(https://crbug.com/webrtc/11393): Can this be renamed? Can this be @@ -171,18 +205,21 @@ class VideoStreamAdapter { Adaptation::StepType step_type_; }; + SequenceChecker sequence_checker_ RTC_GUARDED_BY(&sequence_checker_); // Owner and modifier of the VideoSourceRestriction of this stream adaptor. - const std::unique_ptr source_restrictor_; + const std::unique_ptr source_restrictor_ + RTC_GUARDED_BY(&sequence_checker_); // Decides the next adaptation target in DegradationPreference::BALANCED. const BalancedDegradationSettings balanced_settings_; // To guard against applying adaptations that have become invalidated, an // Adaptation that is applied has to have a matching validation ID. - int adaptation_validation_id_; + int adaptation_validation_id_ RTC_GUARDED_BY(&sequence_checker_); // When deciding the next target up or down, different strategies are used // depending on the DegradationPreference. // https://w3c.github.io/mst-content-hint/#dom-rtcdegradationpreference - DegradationPreference degradation_preference_; - VideoStreamInputState input_state_; + DegradationPreference degradation_preference_ + RTC_GUARDED_BY(&sequence_checker_); + VideoStreamInputState input_state_ RTC_GUARDED_BY(&sequence_checker_); // The input frame rate, resolution and adaptation direction of the last // ApplyAdaptationTarget(). Used to avoid adapting twice if a recent // adaptation has not had an effect on the input frame rate or resolution yet. @@ -190,7 +227,16 @@ class VideoStreamAdapter { // resources intead? If we already have adapted it seems like we should wait // a while before adapting again, so that we are not acting on usage // measurements that are made obsolete/unreliable by an "ongoing" adaptation. - absl::optional last_adaptation_request_; + absl::optional last_adaptation_request_ + RTC_GUARDED_BY(&sequence_checker_); + // The previous restrictions value. Starts as unrestricted. + VideoSourceRestrictions last_video_source_restrictions_ + RTC_GUARDED_BY(&sequence_checker_); + VideoSourceRestrictions last_filtered_restrictions_ + RTC_GUARDED_BY(&sequence_checker_); + + std::vector restrictions_listeners_ + RTC_GUARDED_BY(&sequence_checker_); }; } // namespace webrtc diff --git a/call/adaptation/video_stream_adapter_unittest.cc b/call/adaptation/video_stream_adapter_unittest.cc index 49b291c356..ebb0a0f10a 100644 --- a/call/adaptation/video_stream_adapter_unittest.cc +++ b/call/adaptation/video_stream_adapter_unittest.cc @@ -85,7 +85,7 @@ class FakeVideoStream { // Performs ApplyAdaptation() followed by SetInput() with input pixels and // frame rate adjusted according to the resulting restrictions. void ApplyAdaptation(Adaptation adaptation) { - adapter_->ApplyAdaptation(adaptation); + adapter_->ApplyAdaptation(adaptation, nullptr); // Update input pixels and fps according to the resulting restrictions. auto restrictions = adapter_->source_restrictions(); if (restrictions.target_pixels_per_frame().has_value()) { @@ -110,6 +110,28 @@ class FakeVideoStream { int min_pixels_per_frame_; }; +class FakeVideoStreamAdapterListner : public VideoSourceRestrictionsListener { + public: + void OnVideoSourceRestrictionsUpdated( + VideoSourceRestrictions restrictions, + const VideoAdaptationCounters& adaptation_counters, + rtc::scoped_refptr reason, + const VideoSourceRestrictions& unfiltered_restrictions) { + calls_++; + last_restrictions_ = unfiltered_restrictions; + } + + int calls() const { return calls_; } + + VideoSourceRestrictions last_restrictions() const { + return last_restrictions_; + } + + private: + int calls_ = 0; + VideoSourceRestrictions last_restrictions_; +}; + } // namespace TEST(VideoStreamAdapterTest, NoRestrictionsByDefault) { @@ -126,7 +148,7 @@ TEST(VideoStreamAdapterTest, MaintainFramerate_DecreasesPixelsToThreeFifths) { Adaptation adaptation = adapter.GetAdaptationDown(); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); EXPECT_FALSE(adaptation.min_pixel_limit_reached()); - adapter.ApplyAdaptation(adaptation); + adapter.ApplyAdaptation(adaptation, nullptr); EXPECT_EQ(static_cast((kInputPixels * 3) / 5), adapter.source_restrictions().max_pixels_per_frame()); EXPECT_EQ(absl::nullopt, @@ -195,7 +217,7 @@ TEST(VideoStreamAdapterTest, MaintainResolution_DecreasesFpsToTwoThirds) { InputState(1280 * 720, kInputFps, kDefaultMinPixelsPerFrame)); Adaptation adaptation = adapter.GetAdaptationDown(); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); - adapter.ApplyAdaptation(adaptation); + adapter.ApplyAdaptation(adaptation, nullptr); EXPECT_EQ(absl::nullopt, adapter.source_restrictions().max_pixels_per_frame()); EXPECT_EQ(absl::nullopt, @@ -275,7 +297,7 @@ TEST(VideoStreamAdapterTest, Balanced_DecreaseFrameRate) { // resolution: kBalancedMediumFrameRateFps. Adaptation adaptation = adapter.GetAdaptationDown(); EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); - adapter.ApplyAdaptation(adaptation); + adapter.ApplyAdaptation(adaptation, nullptr); EXPECT_EQ(absl::nullopt, adapter.source_restrictions().max_pixels_per_frame()); EXPECT_EQ(absl::nullopt, @@ -519,7 +541,7 @@ TEST(VideoStreamAdapterTest, MaintainFramerate_AwaitingPreviousAdaptationDown) { adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); adapter.SetInput(InputState(1280 * 720, 30, kDefaultMinPixelsPerFrame)); // Adapt down once, but don't update the input. - adapter.ApplyAdaptation(adapter.GetAdaptationDown()); + adapter.ApplyAdaptation(adapter.GetAdaptationDown(), nullptr); EXPECT_EQ(1, adapter.adaptation_counters().resolution_adaptations); { // Having performed the adaptation, but not updated the input based on the @@ -541,7 +563,7 @@ TEST(VideoStreamAdapterTest, MaintainFramerate_AwaitingPreviousAdaptationUp) { fake_stream.ApplyAdaptation(adapter.GetAdaptationDown()); EXPECT_EQ(2, adapter.adaptation_counters().resolution_adaptations); // Adapt up once, but don't update the input. - adapter.ApplyAdaptation(adapter.GetAdaptationUp()); + adapter.ApplyAdaptation(adapter.GetAdaptationUp(), nullptr); EXPECT_EQ(1, adapter.adaptation_counters().resolution_adaptations); { // Having performed the adaptation, but not updated the input based on the @@ -614,7 +636,7 @@ TEST(VideoStreamAdapterTest, adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); fake_stream.ApplyAdaptation(adapter.GetAdaptationDown()); // Apply adaptation up but don't update input. - adapter.ApplyAdaptation(adapter.GetAdaptationUp()); + adapter.ApplyAdaptation(adapter.GetAdaptationUp(), nullptr); EXPECT_EQ(Adaptation::Status::kAwaitingPreviousAdaptation, adapter.GetAdaptationUp().status()); @@ -668,7 +690,7 @@ TEST(VideoStreamAdapterTest, FakeVideoStream fake_stream(&adapter, 1280 * 720, 30, kDefaultMinPixelsPerFrame); // Apply adaptation but don't update the input. - adapter.ApplyAdaptation(adapter.GetAdaptationDown()); + adapter.ApplyAdaptation(adapter.GetAdaptationDown(), nullptr); EXPECT_EQ(Adaptation::Status::kAwaitingPreviousAdaptation, adapter.GetAdaptationDown().status()); adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); @@ -676,6 +698,36 @@ TEST(VideoStreamAdapterTest, EXPECT_EQ(Adaptation::Status::kValid, adaptation.status()); } +TEST(VideoStreamAdapterTest, RestrictionsBroadcasted) { + VideoStreamAdapter adapter; + FakeVideoStreamAdapterListner listener; + adapter.AddRestrictionsListener(&listener); + adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + FakeVideoStream fake_stream(&adapter, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); + // Not broadcast on invalid ApplyAdaptation. + { + Adaptation adaptation = adapter.GetAdaptationUp(); + adapter.ApplyAdaptation(adaptation, nullptr); + EXPECT_EQ(0, listener.calls()); + } + + // Broadcast on ApplyAdaptation. + { + Adaptation adaptation = adapter.GetAdaptationDown(); + VideoStreamAdapter::RestrictionsWithCounters peek = + adapter.PeekNextRestrictions(adaptation); + fake_stream.ApplyAdaptation(adaptation); + EXPECT_EQ(1, listener.calls()); + EXPECT_EQ(peek.restrictions, listener.last_restrictions()); + } + + // Broadcast on ClearRestrictions(). + adapter.ClearRestrictions(); + EXPECT_EQ(2, listener.calls()); + EXPECT_EQ(VideoSourceRestrictions(), listener.last_restrictions()); +} + TEST(VideoStreamAdapterTest, PeekNextRestrictions) { VideoStreamAdapter adapter; // Any non-disabled DegradationPreference will do. @@ -718,12 +770,24 @@ TEST(VideoStreamAdapterTest, PeekNextRestrictions) { } } +TEST(VideoStreamAdapterTest, PeekRestrictionsDoesNotBroadcast) { + VideoStreamAdapter adapter; + FakeVideoStreamAdapterListner listener; + adapter.AddRestrictionsListener(&listener); + adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); + FakeVideoStream fake_stream(&adapter, 1280 * 720, 30, + kDefaultMinPixelsPerFrame); + Adaptation adaptation = adapter.GetAdaptationDown(); + adapter.PeekNextRestrictions(adaptation); + EXPECT_EQ(0, listener.calls()); +} + TEST(VideoStreamAdapterTest, SetDegradationPreferenceToOrFromBalancedClearsRestrictions) { VideoStreamAdapter adapter; adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_FRAMERATE); adapter.SetInput(InputState(1280 * 720, 30, kDefaultMinPixelsPerFrame)); - adapter.ApplyAdaptation(adapter.GetAdaptationDown()); + adapter.ApplyAdaptation(adapter.GetAdaptationDown(), nullptr); EXPECT_NE(VideoSourceRestrictions(), adapter.source_restrictions()); EXPECT_NE(0, adapter.adaptation_counters().Total()); // Changing from non-balanced to balanced clears the restrictions. @@ -731,7 +795,7 @@ TEST(VideoStreamAdapterTest, EXPECT_EQ(VideoSourceRestrictions(), adapter.source_restrictions()); EXPECT_EQ(0, adapter.adaptation_counters().Total()); // Apply adaptation again. - adapter.ApplyAdaptation(adapter.GetAdaptationDown()); + adapter.ApplyAdaptation(adapter.GetAdaptationDown(), nullptr); EXPECT_NE(VideoSourceRestrictions(), adapter.source_restrictions()); EXPECT_NE(0, adapter.adaptation_counters().Total()); // Changing from balanced to non-balanced clears the restrictions. @@ -752,7 +816,7 @@ TEST(VideoStreamAdapterDeathTest, adapter.SetInput(InputState(1280 * 720, 30, kDefaultMinPixelsPerFrame)); Adaptation adaptation = adapter.GetAdaptationDown(); adapter.SetDegradationPreference(DegradationPreference::MAINTAIN_RESOLUTION); - EXPECT_DEATH(adapter.ApplyAdaptation(adaptation), ""); + EXPECT_DEATH(adapter.ApplyAdaptation(adaptation, nullptr), ""); } TEST(VideoStreamAdapterDeathTest, SetInputInvalidatesAdaptations) { diff --git a/video/adaptation/video_stream_encoder_resource_manager.cc b/video/adaptation/video_stream_encoder_resource_manager.cc index 450b10f8bc..e5ca4912e2 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.cc +++ b/video/adaptation/video_stream_encoder_resource_manager.cc @@ -316,11 +316,13 @@ void VideoStreamEncoderResourceManager::Initialize( } void VideoStreamEncoderResourceManager::SetAdaptationProcessor( - ResourceAdaptationProcessorInterface* adaptation_processor) { + ResourceAdaptationProcessorInterface* adaptation_processor, + VideoStreamAdapter* stream_adapter) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); adaptation_processor_ = adaptation_processor; balanced_constraint_->SetAdaptationProcessor(adaptation_processor); quality_scaler_resource_->SetAdaptationProcessor(adaptation_processor); + stream_adapter_ = stream_adapter; } void VideoStreamEncoderResourceManager::SetDegradationPreferences( @@ -596,7 +598,8 @@ int VideoStreamEncoderResourceManager::LastInputFrameSizeOrDefault() const { void VideoStreamEncoderResourceManager::OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions, const VideoAdaptationCounters& adaptation_counters, - rtc::scoped_refptr reason) { + rtc::scoped_refptr reason, + const VideoSourceRestrictions& unfiltered_restrictions) { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); // TODO(bugs.webrtc.org/11553) Remove reason parameter and add reset callback. if (!reason && adaptation_counters.Total() == 0) { @@ -608,7 +611,8 @@ void VideoStreamEncoderResourceManager::OnVideoSourceRestrictionsUpdated( // means that if the task gets executed, |this| has not been freed yet. encoder_queue_->PostTask([this, restrictions] { RTC_DCHECK_RUN_ON(encoder_queue_); - video_source_restrictions_ = restrictions; + video_source_restrictions_ = FilterRestrictionsByDegradationPreference( + restrictions, degradation_preference_); MaybeUpdateTargetFrameRate(); }); } @@ -720,13 +724,12 @@ void VideoStreamEncoderResourceManager::OnQualityRampUp() { // the adaptation queue, add logic to prevent use-after-free on |this|. resource_adaptation_queue_->PostTask([this] { RTC_DCHECK_RUN_ON(resource_adaptation_queue_); - if (!adaptation_processor_) { + if (!stream_adapter_) { // The processor nulled before this task had a chance to execute. This // happens if the processor is destroyed. No action needed. return; } - RTC_LOG(LS_INFO) << "Reset quality limitations."; - adaptation_processor_->ResetVideoSourceRestrictions(); + stream_adapter_->ClearRestrictions(); }); quality_rampup_experiment_.reset(); } diff --git a/video/adaptation/video_stream_encoder_resource_manager.h b/video/adaptation/video_stream_encoder_resource_manager.h index 61ae29b6bf..edd2491d9d 100644 --- a/video/adaptation/video_stream_encoder_resource_manager.h +++ b/video/adaptation/video_stream_encoder_resource_manager.h @@ -65,6 +65,7 @@ extern const int kDefaultInputPixelsHeight; // ResourceAdaptationProcessor code such as the inital frame dropping. class VideoStreamEncoderResourceManager : public VideoSourceRestrictionsListener, + public ResourceLimitationsListener, public QualityRampUpExperimentListener { public: VideoStreamEncoderResourceManager( @@ -78,7 +79,8 @@ class VideoStreamEncoderResourceManager void Initialize(rtc::TaskQueue* encoder_queue, rtc::TaskQueue* resource_adaptation_queue); void SetAdaptationProcessor( - ResourceAdaptationProcessorInterface* adaptation_processor); + ResourceAdaptationProcessorInterface* adaptation_processor, + VideoStreamAdapter* stream_adapter); // TODO(https://crbug.com/webrtc/11563): The degradation preference is a // setting of the Processor, it does not belong to the Manager - can we get @@ -130,7 +132,8 @@ class VideoStreamEncoderResourceManager void OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions, const VideoAdaptationCounters& adaptation_counters, - rtc::scoped_refptr reason) override; + rtc::scoped_refptr reason, + const VideoSourceRestrictions& unfiltered_restrictions) override; void OnResourceLimitationChanged( rtc::scoped_refptr resource, const std::map, VideoAdaptationCounters>& @@ -238,6 +241,8 @@ class VideoStreamEncoderResourceManager RTC_GUARDED_BY(encoder_queue_); ResourceAdaptationProcessorInterface* adaptation_processor_ RTC_GUARDED_BY(resource_adaptation_queue_); + VideoStreamAdapter* stream_adapter_ + RTC_GUARDED_BY(resource_adaptation_queue_); // Thread-safe. VideoStreamEncoderObserver* const encoder_stats_observer_; diff --git a/video/video_stream_encoder.cc b/video/video_stream_encoder.cc index 0ed73a3e63..53e189123a 100644 --- a/video/video_stream_encoder.cc +++ b/video/video_stream_encoder.cc @@ -256,10 +256,12 @@ VideoStreamEncoder::VideoStreamEncoder( ParseAutomatincAnimationDetectionFieldTrial()), encoder_switch_requested_(false), input_state_provider_(encoder_stats_observer), + video_stream_adapter_(std::make_unique()), resource_adaptation_processor_( std::make_unique( &input_state_provider_, - encoder_stats_observer)), + encoder_stats_observer, + video_stream_adapter_.get())), adaptation_constraints_(), adaptation_listeners_(), stream_resource_manager_(&input_state_provider_, @@ -287,10 +289,11 @@ VideoStreamEncoder::VideoStreamEncoder( resource_adaptation_processor_->SetResourceAdaptationQueue( resource_adaptation_queue_.Get()); stream_resource_manager_.SetAdaptationProcessor( - resource_adaptation_processor_.get()); - resource_adaptation_processor_->AddRestrictionsListener( + resource_adaptation_processor_.get(), video_stream_adapter_.get()); + resource_adaptation_processor_->AddResourceLimitationsListener( &stream_resource_manager_); - resource_adaptation_processor_->AddRestrictionsListener(this); + video_stream_adapter_->AddRestrictionsListener(&stream_resource_manager_); + video_stream_adapter_->AddRestrictionsListener(this); // Add the stream resource manager's resources to the processor. adaptation_constraints_ = stream_resource_manager_.AdaptationConstraints(); @@ -333,10 +336,12 @@ void VideoStreamEncoder::Stop() { for (auto* listener : adaptation_listeners_) { resource_adaptation_processor_->RemoveAdaptationListener(listener); } - resource_adaptation_processor_->RemoveRestrictionsListener(this); - resource_adaptation_processor_->RemoveRestrictionsListener( + video_stream_adapter_->RemoveRestrictionsListener(this); + video_stream_adapter_->RemoveRestrictionsListener( &stream_resource_manager_); - stream_resource_manager_.SetAdaptationProcessor(nullptr); + resource_adaptation_processor_->RemoveResourceLimitationsListener( + &stream_resource_manager_); + stream_resource_manager_.SetAdaptationProcessor(nullptr, nullptr); resource_adaptation_processor_.reset(); } shutdown_adaptation_processor_event.Set(); @@ -1759,7 +1764,8 @@ bool VideoStreamEncoder::DropDueToSize(uint32_t pixel_count) const { void VideoStreamEncoder::OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions, const VideoAdaptationCounters& adaptation_counters, - rtc::scoped_refptr reason) { + rtc::scoped_refptr reason, + const VideoSourceRestrictions& unfiltered_restrictions) { RTC_DCHECK_RUN_ON(&resource_adaptation_queue_); std::string resource_name = reason ? reason->Name() : ""; RTC_LOG(INFO) << "Updating sink restrictions from " << resource_name << " to " @@ -2102,8 +2108,7 @@ void VideoStreamEncoder::AddRestrictionsListenerForTesting( resource_adaptation_queue_.PostTask([this, restrictions_listener, &event] { RTC_DCHECK_RUN_ON(&resource_adaptation_queue_); RTC_DCHECK(resource_adaptation_processor_); - resource_adaptation_processor_->AddRestrictionsListener( - restrictions_listener); + video_stream_adapter_->AddRestrictionsListener(restrictions_listener); event.Set(); }); event.Wait(rtc::Event::kForever); @@ -2115,8 +2120,7 @@ void VideoStreamEncoder::RemoveRestrictionsListenerForTesting( resource_adaptation_queue_.PostTask([this, restrictions_listener, &event] { RTC_DCHECK_RUN_ON(&resource_adaptation_queue_); RTC_DCHECK(resource_adaptation_processor_); - resource_adaptation_processor_->RemoveRestrictionsListener( - restrictions_listener); + video_stream_adapter_->RemoveRestrictionsListener(restrictions_listener); event.Set(); }); event.Wait(rtc::Event::kForever); diff --git a/video/video_stream_encoder.h b/video/video_stream_encoder.h index 68b264deac..7bff722441 100644 --- a/video/video_stream_encoder.h +++ b/video/video_stream_encoder.h @@ -120,7 +120,8 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, void OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions, const VideoAdaptationCounters& adaptation_counters, - rtc::scoped_refptr reason) override; + rtc::scoped_refptr reason, + const VideoSourceRestrictions& unfiltered_restrictions) override; // Used for injected test resources. // TODO(eshr): Move all adaptation tests out of VideoStreamEncoder tests. @@ -408,6 +409,9 @@ class VideoStreamEncoder : public VideoStreamEncoderInterface, // Provies video stream input states: current resolution and frame rate. // This class is thread-safe. VideoStreamInputStateProvider input_state_provider_; + + std::unique_ptr video_stream_adapter_ + RTC_GUARDED_BY(&resource_adaptation_queue_); // Responsible for adapting input resolution or frame rate to ensure resources // (e.g. CPU or bandwidth) are not overused. // This class is single-threaded on the resource adaptation queue. diff --git a/video/video_stream_encoder_unittest.cc b/video/video_stream_encoder_unittest.cc index e963619e60..138c47569b 100644 --- a/video/video_stream_encoder_unittest.cc +++ b/video/video_stream_encoder_unittest.cc @@ -205,7 +205,8 @@ class FakeVideoSourceRestrictionsListener void OnVideoSourceRestrictionsUpdated( VideoSourceRestrictions restrictions, const VideoAdaptationCounters& adaptation_counters, - rtc::scoped_refptr reason) override { + rtc::scoped_refptr reason, + const VideoSourceRestrictions& unfiltered_restrictions) override { was_restrictions_updated_ = true; restrictions_updated_event_.Set(); }