mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-13 05:40:42 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Reland "FrameCadenceAdapter keep track of Input framerate"

Browse source 
This reverts commit d427e83a15.

Reason for revert: Flaky test fixed.

Refactor FrameCandenceAdapter to keep track of input frame rate. This fixes an issue where frame rate is calculated too low if congestion window drop a frame.

Also a field trial WebRTC-FrameCadenceAdapter-UseVideoFrameTimestamp is added to control if VideoFrame timestamp should be used or local clock when calculating frame rate.
Uma is recorded to tell if input frame timestamp is monotonically increasing.

Bug: webrtc:10481, webrtc:15887, webrtc:15893
Change-Id: I76268aa0991dbc99c1b881fb251a76aa54ff2673
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/344561
Reviewed-by: Erik Språng <sprang@google.com>
Commit-Queue: Per Kjellander <perkj@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#41972}
This commit is contained in:
Per K 2024-03-27 11:28:44 +01:00 committed by WebRTC LUCI CQ
parent 15e46aa358
commit e975b44a45
8 changed files with 149 additions and 76 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
experiments/field_trials.py
View file

@ -80,6 +80,9 @@ ACTIVE_FIELD_TRIALS: FrozenSet[FieldTrial] = frozenset([
FieldTrial('WebRTC-EncoderDataDumpDirectory', FieldTrial('WebRTC-EncoderDataDumpDirectory',
'b/296242528', 'b/296242528',
date(2024, 4, 1)), date(2024, 4, 1)),
FieldTrial('WebRTC-FrameCadenceAdapter-UseVideoFrameTimestamp',
'webrtc:15887',
date(2024, 10, 1)),
FieldTrial('WebRTC-IPv6NetworkResolutionFixes', FieldTrial('WebRTC-IPv6NetworkResolutionFixes',
'webrtc:14334', 'webrtc:14334',
date(2024, 4, 1)), date(2024, 4, 1)),

3
modules/congestion_controller/pcc/pcc_network_controller_unittest.cc
View file

@ -112,7 +112,8 @@ TEST(PccNetworkControllerTest, UpdatesTargetSendRate) {
ret_net->UpdateConfig( ret_net->UpdateConfig(
[](NetworkSimulationConfig* c) { c->delay = TimeDelta::Millis(200); }); [](NetworkSimulationConfig* c) { c->delay = TimeDelta::Millis(200); });
s.RunFor(TimeDelta::Seconds(35)); s.RunFor(TimeDelta::Seconds(35));
EXPECT_NEAR(client->target_rate().kbps(), 170, 50); EXPECT_LE(client->target_rate().kbps(), 200);
EXPECT_GT(client->target_rate().kbps(), 90);
} }
} // namespace test } // namespace test

5
test/scenario/video_stream_unittest.cc
View file

@ -248,12 +248,9 @@ TEST(VideoStreamTest, ResolutionAdaptsToAvailableBandwidth) {
num_qvga_frames_ = 0; num_qvga_frames_ = 0;
num_vga_frames_ = 0; num_vga_frames_ = 0;
s.RunFor(TimeDelta::Seconds(40)); s.RunFor(TimeDelta::Seconds(70));
EXPECT_GT(num_qvga_frames_, 0u); EXPECT_GT(num_qvga_frames_, 0u);
#ifndef __ANDROID__
// TODO: crbug.com/webrtc/15873 - This expectation is flaky on Android.
EXPECT_GT(num_vga_frames_, 0u); EXPECT_GT(num_vga_frames_, 0u);
#endif
} }
TEST(VideoStreamTest, SuspendsBelowMinBitrate) { TEST(VideoStreamTest, SuspendsBelowMinBitrate) {

71
video/frame_cadence_adapter.cc
View file

@ -12,6 +12,7 @@
#include <algorithm> #include <algorithm>
#include <atomic> #include <atomic>
#include <cstdint>
#include <deque> #include <deque>
#include <memory> #include <memory>
#include <utility> #include <utility>
@ -58,15 +59,15 @@ class AdapterMode {
virtual absl::optional<uint32_t> GetInputFrameRateFps() = 0; virtual absl::optional<uint32_t> GetInputFrameRateFps() = 0;
// Updates the frame rate. // Updates the frame rate.
virtual void UpdateFrameRate() = 0; virtual void UpdateFrameRate(Timestamp frame_timestamp) = 0;
}; };
// Implements a pass-through adapter. Single-threaded. // Implements a pass-through adapter. Single-threaded.
class PassthroughAdapterMode : public AdapterMode { class PassthroughAdapterMode : public AdapterMode {
public: public:
PassthroughAdapterMode(Clock* clock, explicit PassthroughAdapterMode(
FrameCadenceAdapterInterface::Callback* callback) FrameCadenceAdapterInterface::Callback* callback)
: clock_(clock), callback_(callback) { : callback_(callback) {
sequence_checker_.Detach(); sequence_checker_.Detach();
} }
@ -80,16 +81,18 @@ class PassthroughAdapterMode : public AdapterMode {
absl::optional<uint32_t> GetInputFrameRateFps() override { absl::optional<uint32_t> GetInputFrameRateFps() override {
RTC_DCHECK_RUN_ON(&sequence_checker_); RTC_DCHECK_RUN_ON(&sequence_checker_);
return input_framerate_.Rate(clock_->TimeInMilliseconds()); return last_frame_rate_;
} }
void UpdateFrameRate() override { void UpdateFrameRate(Timestamp frame_timestamp) override {
RTC_DCHECK_RUN_ON(&sequence_checker_); RTC_DCHECK_RUN_ON(&sequence_checker_);
input_framerate_.Update(1, clock_->TimeInMilliseconds()); // RateStatistics will calculate a too high rate immediately after Update.
last_frame_rate_ = input_framerate_.Rate(frame_timestamp.ms());
input_framerate_.Update(1, frame_timestamp.ms());
} }
private: private:
Clock* const clock_; absl::optional<uint64_t> last_frame_rate_;
FrameCadenceAdapterInterface::Callback* const callback_; FrameCadenceAdapterInterface::Callback* const callback_;
RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_; RTC_NO_UNIQUE_ADDRESS SequenceChecker sequence_checker_;
// Input frame rate statistics for use when not in zero-hertz mode. // Input frame rate statistics for use when not in zero-hertz mode.
@ -125,7 +128,7 @@ class ZeroHertzAdapterMode : public AdapterMode {
bool queue_overload, bool queue_overload,
const VideoFrame& frame) override; const VideoFrame& frame) override;
absl::optional<uint32_t> GetInputFrameRateFps() override; absl::optional<uint32_t> GetInputFrameRateFps() override;
void UpdateFrameRate() override {} void UpdateFrameRate(Timestamp frame_timestamp) override {}
// Notified on dropped frames. // Notified on dropped frames.
void OnDiscardedFrame(); void OnDiscardedFrame();
@ -283,12 +286,14 @@ class VSyncEncodeAdapterMode : public AdapterMode {
absl::optional<uint32_t> GetInputFrameRateFps() override { absl::optional<uint32_t> GetInputFrameRateFps() override {
RTC_DCHECK_RUN_ON(&queue_sequence_checker_); RTC_DCHECK_RUN_ON(&queue_sequence_checker_);
return input_framerate_.Rate(clock_->TimeInMilliseconds()); return last_frame_rate_;
} }
void UpdateFrameRate() override { void UpdateFrameRate(Timestamp frame_timestamp) override {
RTC_DCHECK_RUN_ON(&queue_sequence_checker_); RTC_DCHECK_RUN_ON(&queue_sequence_checker_);
input_framerate_.Update(1, clock_->TimeInMilliseconds()); // RateStatistics will calculate a too high rate immediately after Update.
last_frame_rate_ = input_framerate_.Rate(frame_timestamp.ms());
input_framerate_.Update(1, frame_timestamp.ms());
} }
void EncodeAllEnqueuedFrames(); void EncodeAllEnqueuedFrames();
@ -308,6 +313,8 @@ class VSyncEncodeAdapterMode : public AdapterMode {
RTC_NO_UNIQUE_ADDRESS SequenceChecker queue_sequence_checker_; RTC_NO_UNIQUE_ADDRESS SequenceChecker queue_sequence_checker_;
rtc::scoped_refptr<PendingTaskSafetyFlag> queue_safety_flag_; rtc::scoped_refptr<PendingTaskSafetyFlag> queue_safety_flag_;
// Input frame rate statistics for use when not in zero-hertz mode. // Input frame rate statistics for use when not in zero-hertz mode.
absl::optional<uint64_t> last_frame_rate_
RTC_GUARDED_BY(queue_sequence_checker_);
RateStatistics input_framerate_ RTC_GUARDED_BY(queue_sequence_checker_){ RateStatistics input_framerate_ RTC_GUARDED_BY(queue_sequence_checker_){
FrameCadenceAdapterInterface::kFrameRateAveragingWindowSizeMs, 1000}; FrameCadenceAdapterInterface::kFrameRateAveragingWindowSizeMs, 1000};
FrameCadenceAdapterInterface::Callback* const callback_; FrameCadenceAdapterInterface::Callback* const callback_;
@ -315,8 +322,8 @@ class VSyncEncodeAdapterMode : public AdapterMode {
Metronome* metronome_; Metronome* metronome_;
TaskQueueBase* const worker_queue_; TaskQueueBase* const worker_queue_;
RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_sequence_checker_; RTC_NO_UNIQUE_ADDRESS SequenceChecker worker_sequence_checker_;
// `worker_safety_` protects tasks on the worker queue related to `metronome_` // `worker_safety_` protects tasks on the worker queue related to
// since metronome usage must happen on worker thread. // `metronome_` since metronome usage must happen on worker thread.
ScopedTaskSafetyDetached worker_safety_; ScopedTaskSafetyDetached worker_safety_;
Timestamp expected_next_tick_ RTC_GUARDED_BY(worker_sequence_checker_) = Timestamp expected_next_tick_ RTC_GUARDED_BY(worker_sequence_checker_) =
Timestamp::PlusInfinity(); Timestamp::PlusInfinity();
@ -339,7 +346,6 @@ class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface {
void SetZeroHertzModeEnabled( void SetZeroHertzModeEnabled(
absl::optional<ZeroHertzModeParams> params) override; absl::optional<ZeroHertzModeParams> params) override;
absl::optional<uint32_t> GetInputFrameRateFps() override; absl::optional<uint32_t> GetInputFrameRateFps() override;
void UpdateFrameRate() override;
void UpdateLayerQualityConvergence(size_t spatial_index, void UpdateLayerQualityConvergence(size_t spatial_index,
bool quality_converged) override; bool quality_converged) override;
void UpdateLayerStatus(size_t spatial_index, bool enabled) override; void UpdateLayerStatus(size_t spatial_index, bool enabled) override;
@ -354,6 +360,7 @@ class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface {
const VideoTrackSourceConstraints& constraints) override; const VideoTrackSourceConstraints& constraints) override;
private: private:
void UpdateFrameRate(Timestamp frame_timestamp);
// Called from OnFrame in both pass-through and zero-hertz mode. // Called from OnFrame in both pass-through and zero-hertz mode.
void OnFrameOnMainQueue(Timestamp post_time, void OnFrameOnMainQueue(Timestamp post_time,
bool queue_overload, bool queue_overload,
@ -383,6 +390,13 @@ class FrameCadenceAdapterImpl : public FrameCadenceAdapterInterface {
// Kill-switch for the queue overload mechanism in zero-hertz mode. // Kill-switch for the queue overload mechanism in zero-hertz mode.
const bool frame_cadence_adapter_zero_hertz_queue_overload_enabled_; const bool frame_cadence_adapter_zero_hertz_queue_overload_enabled_;
// Field trial for using timestamp from video frames, rather than clock when
// calculating input frame rate.
const bool use_video_frame_timestamp_;
// Used for verifying that timestamps are monotonically increasing.
absl::optional<Timestamp> last_incoming_frame_timestamp_;
bool incoming_frame_timestamp_monotonically_increasing_ = true;
// The three possible modes we're under. // The three possible modes we're under.
absl::optional<PassthroughAdapterMode> passthrough_adapter_; absl::optional<PassthroughAdapterMode> passthrough_adapter_;
absl::optional<ZeroHertzAdapterMode> zero_hertz_adapter_; absl::optional<ZeroHertzAdapterMode> zero_hertz_adapter_;
@ -837,6 +851,8 @@ FrameCadenceAdapterImpl::FrameCadenceAdapterImpl(
!field_trials.IsDisabled("WebRTC-ZeroHertzScreenshare")), !field_trials.IsDisabled("WebRTC-ZeroHertzScreenshare")),
frame_cadence_adapter_zero_hertz_queue_overload_enabled_( frame_cadence_adapter_zero_hertz_queue_overload_enabled_(
!field_trials.IsDisabled("WebRTC-ZeroHertzQueueOverload")), !field_trials.IsDisabled("WebRTC-ZeroHertzQueueOverload")),
use_video_frame_timestamp_(field_trials.IsEnabled(
"WebRTC-FrameCadenceAdapter-UseVideoFrameTimestamp")),
metronome_(metronome), metronome_(metronome),
worker_queue_(worker_queue) {} worker_queue_(worker_queue) {}
@ -849,6 +865,10 @@ FrameCadenceAdapterImpl::~FrameCadenceAdapterImpl() {
absl::Cleanup cleanup = [adapter = std::move(vsync_encode_adapter_)] {}; absl::Cleanup cleanup = [adapter = std::move(vsync_encode_adapter_)] {};
worker_queue_->PostTask([cleanup = std::move(cleanup)] {}); worker_queue_->PostTask([cleanup = std::move(cleanup)] {});
} }
RTC_HISTOGRAM_BOOLEAN(
"WebRTC.Video.InputFrameTimestampMonotonicallyIncreasing",
incoming_frame_timestamp_monotonically_increasing_);
} }
void FrameCadenceAdapterImpl::Initialize(Callback* callback) { void FrameCadenceAdapterImpl::Initialize(Callback* callback) {
@ -859,7 +879,7 @@ void FrameCadenceAdapterImpl::Initialize(Callback* callback) {
vsync_encode_adapter_ = std::make_unique<VSyncEncodeAdapterMode>( vsync_encode_adapter_ = std::make_unique<VSyncEncodeAdapterMode>(
clock_, queue_, safety_.flag(), metronome_, worker_queue_, callback_); clock_, queue_, safety_.flag(), metronome_, worker_queue_, callback_);
} else { } else {
passthrough_adapter_.emplace(clock_, callback); passthrough_adapter_.emplace(callback);
} }
ConfigureCurrentAdapterWithoutZeroHertz(); ConfigureCurrentAdapterWithoutZeroHertz();
} }
@ -877,7 +897,7 @@ absl::optional<uint32_t> FrameCadenceAdapterImpl::GetInputFrameRateFps() {
return current_adapter_mode_->GetInputFrameRateFps(); return current_adapter_mode_->GetInputFrameRateFps();
} }
void FrameCadenceAdapterImpl::UpdateFrameRate() { void FrameCadenceAdapterImpl::UpdateFrameRate(Timestamp frame_timestamp) {
RTC_DCHECK_RUN_ON(queue_); RTC_DCHECK_RUN_ON(queue_);
// The frame rate need not be updated for the zero-hertz adapter. The // The frame rate need not be updated for the zero-hertz adapter. The
// vsync encode and passthrough adapter however uses it. Always pass frames // vsync encode and passthrough adapter however uses it. Always pass frames
@ -885,10 +905,10 @@ void FrameCadenceAdapterImpl::UpdateFrameRate() {
// there be an adapter switch. // there be an adapter switch.
if (metronome_) { if (metronome_) {
RTC_CHECK(vsync_encode_adapter_); RTC_CHECK(vsync_encode_adapter_);
vsync_encode_adapter_->UpdateFrameRate(); vsync_encode_adapter_->UpdateFrameRate(frame_timestamp);
} else { } else {
RTC_CHECK(passthrough_adapter_); RTC_CHECK(passthrough_adapter_);
passthrough_adapter_->UpdateFrameRate(); passthrough_adapter_->UpdateFrameRate(frame_timestamp);
} }
} }
@ -979,6 +999,19 @@ void FrameCadenceAdapterImpl::OnFrameOnMainQueue(Timestamp post_time,
const VideoFrame& frame) { const VideoFrame& frame) {
RTC_DCHECK_RUN_ON(queue_); RTC_DCHECK_RUN_ON(queue_);
current_adapter_mode_->OnFrame(post_time, queue_overload, frame); current_adapter_mode_->OnFrame(post_time, queue_overload, frame);
if (last_incoming_frame_timestamp_ &&
last_incoming_frame_timestamp_ >=
Timestamp::Micros(frame.timestamp_us())) {
RTC_LOG(LS_ERROR)
<< "Incoming frame timestamp is not monotonically increasing"
<< " current: " << frame.timestamp_us()
<< " last: " << last_incoming_frame_timestamp_.value().us();
incoming_frame_timestamp_monotonically_increasing_ = false;
}
last_incoming_frame_timestamp_ = Timestamp::Micros(frame.timestamp_us());
Timestamp update_frame_rate_timestamp =
use_video_frame_timestamp_ ? *last_incoming_frame_timestamp_ : post_time;
UpdateFrameRate(update_frame_rate_timestamp);
} }
bool FrameCadenceAdapterImpl::IsZeroHertzScreenshareEnabled() const { bool FrameCadenceAdapterImpl::IsZeroHertzScreenshareEnabled() const {

4
video/frame_cadence_adapter.h
View file

@ -99,10 +99,6 @@ class FrameCadenceAdapterInterface
// zero-hertz mode is off, and returns the max framerate in zero-hertz mode. // zero-hertz mode is off, and returns the max framerate in zero-hertz mode.
virtual absl::optional<uint32_t> GetInputFrameRateFps() = 0; virtual absl::optional<uint32_t> GetInputFrameRateFps() = 0;
// Updates frame rate. This is done unconditionally irrespective of adapter
// mode.
virtual void UpdateFrameRate() = 0;
// Updates quality convergence status for an enabled spatial layer. // Updates quality convergence status for an enabled spatial layer.
// Convergence means QP has dropped to a low-enough level to warrant ceasing // Convergence means QP has dropped to a low-enough level to warrant ceasing
// to send identical frames at high frequency. // to send identical frames at high frequency.

62
video/frame_cadence_adapter_unittest.cc
View file

@ -10,6 +10,8 @@
#include "video/frame_cadence_adapter.h" #include "video/frame_cadence_adapter.h"
#include <cstdint>
#include <memory>
#include <utility> #include <utility>
#include <vector> #include <vector>
@ -134,7 +136,8 @@ TEST(FrameCadenceAdapterTest, FrameRateFollowsRateStatisticsByDefault) {
test::ScopedKeyValueConfig no_field_trials; test::ScopedKeyValueConfig no_field_trials;
GlobalSimulatedTimeController time_controller(Timestamp::Zero()); GlobalSimulatedTimeController time_controller(Timestamp::Zero());
auto adapter = CreateAdapter(no_field_trials, time_controller.GetClock()); auto adapter = CreateAdapter(no_field_trials, time_controller.GetClock());
adapter->Initialize(nullptr); MockCallback callback;
adapter->Initialize(&callback);
// Create an "oracle" rate statistics which should be followed on a sequence // Create an "oracle" rate statistics which should be followed on a sequence
// of frames. // of frames.
@ -143,10 +146,13 @@ TEST(FrameCadenceAdapterTest, FrameRateFollowsRateStatisticsByDefault) {
for (int frame = 0; frame != 10; ++frame) { for (int frame = 0; frame != 10; ++frame) {
time_controller.AdvanceTime(TimeDelta::Millis(10)); time_controller.AdvanceTime(TimeDelta::Millis(10));
absl::optional<int64_t> expected_fps =
rate.Rate(time_controller.GetClock()->TimeInMilliseconds());
rate.Update(1, time_controller.GetClock()->TimeInMilliseconds()); rate.Update(1, time_controller.GetClock()->TimeInMilliseconds());
adapter->UpdateFrameRate(); // FrameCadanceAdapter::OnFrame post the frame to another sequence.
EXPECT_EQ(rate.Rate(time_controller.GetClock()->TimeInMilliseconds()), adapter->OnFrame(CreateFrameWithTimestamps(&time_controller));
adapter->GetInputFrameRateFps()) time_controller.AdvanceTime(TimeDelta::Millis(0));
EXPECT_EQ(expected_fps, adapter->GetInputFrameRateFps())
<< " failed for frame " << frame; << " failed for frame " << frame;
} }
} }
@ -156,7 +162,8 @@ TEST(FrameCadenceAdapterTest,
ZeroHertzFieldTrialDisabler feature_disabler; ZeroHertzFieldTrialDisabler feature_disabler;
GlobalSimulatedTimeController time_controller(Timestamp::Zero()); GlobalSimulatedTimeController time_controller(Timestamp::Zero());
auto adapter = CreateAdapter(feature_disabler, time_controller.GetClock()); auto adapter = CreateAdapter(feature_disabler, time_controller.GetClock());
adapter->Initialize(nullptr); MockCallback callback;
adapter->Initialize(&callback);
// Create an "oracle" rate statistics which should be followed on a sequence // Create an "oracle" rate statistics which should be followed on a sequence
// of frames. // of frames.
@ -165,10 +172,14 @@ TEST(FrameCadenceAdapterTest,
for (int frame = 0; frame != 10; ++frame) { for (int frame = 0; frame != 10; ++frame) {
time_controller.AdvanceTime(TimeDelta::Millis(10)); time_controller.AdvanceTime(TimeDelta::Millis(10));
absl::optional<int64_t> expected_fps =
rate.Rate(time_controller.GetClock()->TimeInMilliseconds());
rate.Update(1, time_controller.GetClock()->TimeInMilliseconds()); rate.Update(1, time_controller.GetClock()->TimeInMilliseconds());
adapter->UpdateFrameRate(); // FrameCadanceAdapter::OnFrame post the frame to another sequence.
EXPECT_EQ(rate.Rate(time_controller.GetClock()->TimeInMilliseconds()), adapter->OnFrame(CreateFrameWithTimestamps(&time_controller));
adapter->GetInputFrameRateFps()) time_controller.AdvanceTime(TimeDelta::Millis(0));
EXPECT_EQ(adapter->GetInputFrameRateFps(), expected_fps)
<< " failed for frame " << frame; << " failed for frame " << frame;
} }
} }
@ -177,13 +188,16 @@ TEST(FrameCadenceAdapterTest, FrameRateFollowsMaxFpsWhenZeroHertzActivated) {
ZeroHertzFieldTrialEnabler enabler; ZeroHertzFieldTrialEnabler enabler;
GlobalSimulatedTimeController time_controller(Timestamp::Zero()); GlobalSimulatedTimeController time_controller(Timestamp::Zero());
auto adapter = CreateAdapter(enabler, time_controller.GetClock()); auto adapter = CreateAdapter(enabler, time_controller.GetClock());
adapter->Initialize(nullptr); MockCallback callback;
adapter->Initialize(&callback);
adapter->SetZeroHertzModeEnabled( adapter->SetZeroHertzModeEnabled(
FrameCadenceAdapterInterface::ZeroHertzModeParams{}); FrameCadenceAdapterInterface::ZeroHertzModeParams{});
adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1}); adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1});
for (int frame = 0; frame != 10; ++frame) { for (int frame = 0; frame != 10; ++frame) {
time_controller.AdvanceTime(TimeDelta::Millis(10)); time_controller.AdvanceTime(TimeDelta::Millis(10));
adapter->UpdateFrameRate(); // FrameCadanceAdapter::OnFrame post the frame to another sequence.
adapter->OnFrame(CreateFrameWithTimestamps(&time_controller));
time_controller.AdvanceTime(TimeDelta::Millis(0));
EXPECT_EQ(adapter->GetInputFrameRateFps(), 1u); EXPECT_EQ(adapter->GetInputFrameRateFps(), 1u);
} }
} }
@ -216,7 +230,8 @@ TEST(FrameCadenceAdapterTest,
ZeroHertzFieldTrialEnabler enabler; ZeroHertzFieldTrialEnabler enabler;
GlobalSimulatedTimeController time_controller(Timestamp::Zero()); GlobalSimulatedTimeController time_controller(Timestamp::Zero());
auto adapter = CreateAdapter(enabler, time_controller.GetClock()); auto adapter = CreateAdapter(enabler, time_controller.GetClock());
adapter->Initialize(nullptr); MockCallback callback;
adapter->Initialize(&callback);
adapter->SetZeroHertzModeEnabled( adapter->SetZeroHertzModeEnabled(
FrameCadenceAdapterInterface::ZeroHertzModeParams{}); FrameCadenceAdapterInterface::ZeroHertzModeParams{});
adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1}); adapter->OnConstraintsChanged(VideoTrackSourceConstraints{0, 1});
@ -226,15 +241,16 @@ TEST(FrameCadenceAdapterTest,
for (int frame = 0; frame != MAX; ++frame) { for (int frame = 0; frame != MAX; ++frame) {
time_controller.AdvanceTime(TimeDelta::Millis(10)); time_controller.AdvanceTime(TimeDelta::Millis(10));
rate.Update(1, time_controller.GetClock()->TimeInMilliseconds()); rate.Update(1, time_controller.GetClock()->TimeInMilliseconds());
adapter->UpdateFrameRate(); adapter->OnFrame(CreateFrameWithTimestamps(&time_controller));
time_controller.AdvanceTime(TimeDelta::Millis(0));
} }
// Turn off zero hertz on the next-last frame; after the last frame we // Turn off zero hertz on the next-last frame; after the last frame we
// should see a value that tracks the rate oracle. // should see a value that tracks the rate oracle.
adapter->SetZeroHertzModeEnabled(absl::nullopt); adapter->SetZeroHertzModeEnabled(absl::nullopt);
// Last frame. // Last frame.
time_controller.AdvanceTime(TimeDelta::Millis(10)); time_controller.AdvanceTime(TimeDelta::Millis(10));
rate.Update(1, time_controller.GetClock()->TimeInMilliseconds()); adapter->OnFrame(CreateFrameWithTimestamps(&time_controller));
adapter->UpdateFrameRate(); time_controller.AdvanceTime(TimeDelta::Millis(0));
EXPECT_EQ(rate.Rate(time_controller.GetClock()->TimeInMilliseconds()), EXPECT_EQ(rate.Rate(time_controller.GetClock()->TimeInMilliseconds()),
adapter->GetInputFrameRateFps()); adapter->GetInputFrameRateFps());
@ -1038,6 +1054,24 @@ TEST_F(FrameCadenceAdapterMetricsTest, RecordsTimeUntilFirstFrame) {
ElementsAre(Pair(666, 1))); ElementsAre(Pair(666, 1)));
} }
TEST_F(FrameCadenceAdapterMetricsTest,
RecordsFrameTimestampMonotonicallyIncreasing) {
MockCallback callback;
test::ScopedKeyValueConfig no_field_trials;
std::unique_ptr<FrameCadenceAdapterInterface> adapter =
CreateAdapter(no_field_trials, time_controller_.GetClock());
adapter->Initialize(&callback);
time_controller_.AdvanceTime(TimeDelta::Millis(666));
adapter->OnFrame(CreateFrameWithTimestamps(&time_controller_));
adapter->OnFrame(CreateFrameWithTimestamps(&time_controller_));
time_controller_.AdvanceTime(TimeDelta::Zero());
adapter = nullptr;
DepleteTaskQueues();
EXPECT_THAT(metrics::Samples(
"WebRTC.Video.InputFrameTimestampMonotonicallyIncreasing"),
ElementsAre(Pair(false, 1)));
}
TEST(FrameCadenceAdapterRealTimeTest, TimestampsDoNotDrift) { TEST(FrameCadenceAdapterRealTimeTest, TimestampsDoNotDrift) {
// This regression test must be performed in realtime because of limitations // This regression test must be performed in realtime because of limitations
// in GlobalSimulatedTimeController. // in GlobalSimulatedTimeController.

6
video/video_stream_encoder.cc
View file

@ -1784,13 +1784,7 @@ void VideoStreamEncoder::MaybeEncodeVideoFrame(const VideoFrame& video_frame,
// According to the testcase // According to the testcase
// InitialFrameDropOffWhenEncoderDisabledScaling, the return value // InitialFrameDropOffWhenEncoderDisabledScaling, the return value
// from GetScalingSettings should enable or disable the frame drop. // from GetScalingSettings should enable or disable the frame drop.
// Update input frame rate before we start using it. If we update it after
// any potential frame drop we are going to artificially increase frame sizes.
// Poll the rate before updating, otherwise we risk the rate being estimated
// a little too high at the start of the call when then window is small.
uint32_t framerate_fps = GetInputFramerateFps(); uint32_t framerate_fps = GetInputFramerateFps();
frame_cadence_adapter_->UpdateFrameRate();
int64_t now_ms = env_.clock().TimeInMilliseconds(); int64_t now_ms = env_.clock().TimeInMilliseconds();
if (pending_encoder_reconfiguration_) { if (pending_encoder_reconfiguration_) {

71
video/video_stream_encoder_unittest.cc
View file

@ -574,7 +574,7 @@ class AdaptingFrameForwarder : public test::FrameForwarder {
.set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>( .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
nullptr, out_width, out_height)) nullptr, out_width, out_height))
.set_ntp_time_ms(video_frame.ntp_time_ms()) .set_ntp_time_ms(video_frame.ntp_time_ms())
.set_timestamp_ms(99) .set_timestamp_ms(video_frame.timestamp_us() * 1000)
.set_rotation(kVideoRotation_0) .set_rotation(kVideoRotation_0)
.build(); .build();
if (video_frame.has_update_rect()) { if (video_frame.has_update_rect()) {
@ -791,7 +791,6 @@ class MockFrameCadenceAdapter : public FrameCadenceAdapterInterface {
(override)); (override));
MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override)); MOCK_METHOD(void, OnFrame, (const VideoFrame&), (override));
MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override)); MOCK_METHOD(absl::optional<uint32_t>, GetInputFrameRateFps, (), (override));
MOCK_METHOD(void, UpdateFrameRate, (), (override));
MOCK_METHOD(void, MOCK_METHOD(void,
UpdateLayerQualityConvergence, UpdateLayerQualityConvergence,
(size_t spatial_index, bool converged), (size_t spatial_index, bool converged),
@ -952,7 +951,7 @@ class VideoStreamEncoderTest : public ::testing::Test {
.set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>( .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
destruction_event, codec_width_, codec_height_)) destruction_event, codec_width_, codec_height_))
.set_ntp_time_ms(ntp_time_ms) .set_ntp_time_ms(ntp_time_ms)
.set_timestamp_ms(99) .set_timestamp_ms(ntp_time_ms)
.set_rotation(kVideoRotation_0) .set_rotation(kVideoRotation_0)
.build(); .build();
} }
@ -964,7 +963,7 @@ class VideoStreamEncoderTest : public ::testing::Test {
.set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>( .set_video_frame_buffer(rtc::make_ref_counted<TestBuffer>(
destruction_event, codec_width_, codec_height_)) destruction_event, codec_width_, codec_height_))
.set_ntp_time_ms(ntp_time_ms) .set_ntp_time_ms(ntp_time_ms)
.set_timestamp_ms(99) .set_timestamp_ms(ntp_time_ms)
.set_rotation(kVideoRotation_0) .set_rotation(kVideoRotation_0)
.set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1}) .set_update_rect(VideoFrame::UpdateRect{offset_x, 0, 1, 1})
.build(); .build();
@ -998,7 +997,7 @@ class VideoStreamEncoderTest : public ::testing::Test {
.set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>( .set_video_frame_buffer(rtc::make_ref_counted<FakeNativeBuffer>(
destruction_event, width, height)) destruction_event, width, height))
.set_ntp_time_ms(ntp_time_ms) .set_ntp_time_ms(ntp_time_ms)
.set_timestamp_ms(99) .set_timestamp_ms(ntp_time_ms)
.set_rotation(kVideoRotation_0) .set_rotation(kVideoRotation_0)
.build(); .build();
} }
@ -1011,7 +1010,7 @@ class VideoStreamEncoderTest : public ::testing::Test {
.set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>( .set_video_frame_buffer(rtc::make_ref_counted<FakeNV12NativeBuffer>(
destruction_event, width, height)) destruction_event, width, height))
.set_ntp_time_ms(ntp_time_ms) .set_ntp_time_ms(ntp_time_ms)
.set_timestamp_ms(99) .set_timestamp_ms(ntp_time_ms)
.set_rotation(kVideoRotation_0) .set_rotation(kVideoRotation_0)
.build(); .build();
} }
@ -1533,6 +1532,7 @@ class VideoStreamEncoderTest : public ::testing::Test {
if (num_received_layers_ == num_expected_layers_) { if (num_received_layers_ == num_expected_layers_) {
encoded_frame_event_.Set(); encoded_frame_event_.Set();
} }
return Result(Result::OK, last_timestamp_); return Result(Result::OK, last_timestamp_);
} }
@ -1663,28 +1663,34 @@ TEST_F(VideoStreamEncoderTest, DropsFramesBeforeFirstOnBitrateUpdated) {
// The encoder will cache up to one frame for a short duration. Adding two // The encoder will cache up to one frame for a short duration. Adding two
// frames means that the first frame will be dropped and the second frame will // frames means that the first frame will be dropped and the second frame will
// be sent when the encoder is enabled. // be sent when the encoder is enabled.
video_source_.IncomingCapturedFrame(CreateFrame(1, &frame_destroyed_event)); const int64_t kFrame1TimestampMs = CurrentTimeMs();
video_source_.IncomingCapturedFrame(
CreateFrame(kFrame1TimestampMs, &frame_destroyed_event));
AdvanceTime(TimeDelta::Millis(10)); AdvanceTime(TimeDelta::Millis(10));
video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr)); const int64_t kFrame2TimestampMs = CurrentTimeMs();
video_source_.IncomingCapturedFrame(CreateFrame(kFrame2TimestampMs, nullptr));
AdvanceTime(TimeDelta::Zero()); AdvanceTime(TimeDelta::Zero());
EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout)); EXPECT_TRUE(frame_destroyed_event.Wait(kDefaultTimeout));
video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0); kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
// The pending frame should be received. // The pending frame should be encoded.
WaitForEncodedFrame(2); WaitForEncodedFrame(kFrame2TimestampMs);
video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr));
WaitForEncodedFrame(3); const int64_t kFrame3TimestampMs = CurrentTimeMs();
video_source_.IncomingCapturedFrame(CreateFrame(kFrame3TimestampMs, nullptr));
WaitForEncodedFrame(kFrame3TimestampMs);
video_stream_encoder_->Stop(); video_stream_encoder_->Stop();
} }
TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) { TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
int64_t time_ms = 123;
video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0); kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
video_source_.IncomingCapturedFrame(CreateFrame(1, nullptr)); video_source_.IncomingCapturedFrame(CreateFrame(time_ms, nullptr));
WaitForEncodedFrame(1); WaitForEncodedFrame(time_ms);
video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0); DataRate::Zero(), DataRate::Zero(), DataRate::Zero(), 0, 0, 0);
@ -1692,14 +1698,17 @@ TEST_F(VideoStreamEncoderTest, DropsFramesWhenRateSetToZero) {
// The encoder will cache up to one frame for a short duration. Adding two // The encoder will cache up to one frame for a short duration. Adding two
// frames means that the first frame will be dropped and the second frame will // frames means that the first frame will be dropped and the second frame will
// be sent when the encoder is resumed. // be sent when the encoder is resumed.
video_source_.IncomingCapturedFrame(CreateFrame(2, nullptr)); time_ms += 30;
video_source_.IncomingCapturedFrame(CreateFrame(3, nullptr)); video_source_.IncomingCapturedFrame(CreateFrame(time_ms, nullptr));
time_ms += 30;
video_source_.IncomingCapturedFrame(CreateFrame(time_ms, nullptr));
video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0); kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
WaitForEncodedFrame(3); WaitForEncodedFrame(time_ms);
video_source_.IncomingCapturedFrame(CreateFrame(4, nullptr)); time_ms += 20;
WaitForEncodedFrame(4); video_source_.IncomingCapturedFrame(CreateFrame(time_ms, nullptr));
WaitForEncodedFrame(time_ms);
video_stream_encoder_->Stop(); video_stream_encoder_->Stop();
} }
@ -4707,20 +4716,24 @@ TEST_F(VideoStreamEncoderTest,
TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) { TEST_F(VideoStreamEncoderTest, CpuLimitedHistogramIsReported) {
const int kWidth = 640; const int kWidth = 640;
const int kHeight = 360; const int kHeight = 360;
int64_t ntp_timestamp_ms = 123;
video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources( video_stream_encoder_->OnBitrateUpdatedAndWaitForManagedResources(
kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0); kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) { for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight)); video_source_.IncomingCapturedFrame(
WaitForEncodedFrame(i); CreateFrame(ntp_timestamp_ms, kWidth, kHeight));
WaitForEncodedFrame(ntp_timestamp_ms);
ntp_timestamp_ms += 20;
} }
video_stream_encoder_->TriggerCpuOveruse(); video_stream_encoder_->TriggerCpuOveruse();
for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) { for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
video_source_.IncomingCapturedFrame(CreateFrame( video_source_.IncomingCapturedFrame(
SendStatisticsProxy::kMinRequiredMetricsSamples + i, kWidth, kHeight)); CreateFrame(ntp_timestamp_ms, kWidth, kHeight));
WaitForEncodedFrame(SendStatisticsProxy::kMinRequiredMetricsSamples + i); WaitForEncodedFrame(ntp_timestamp_ms);
ntp_timestamp_ms += 20;
} }
video_stream_encoder_->Stop(); video_stream_encoder_->Stop();
@ -4739,13 +4752,16 @@ TEST_F(VideoStreamEncoderTest,
kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0); kTargetBitrate, kTargetBitrate, kTargetBitrate, 0, 0, 0);
const int kWidth = 640; const int kWidth = 640;
const int kHeight = 360; const int kHeight = 360;
int64_t ntp_timestamp_ms = 123;
video_stream_encoder_->SetSource(&video_source_, video_stream_encoder_->SetSource(&video_source_,
webrtc::DegradationPreference::DISABLED); webrtc::DegradationPreference::DISABLED);
for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) { for (int i = 1; i <= SendStatisticsProxy::kMinRequiredMetricsSamples; ++i) {
video_source_.IncomingCapturedFrame(CreateFrame(i, kWidth, kHeight)); video_source_.IncomingCapturedFrame(
WaitForEncodedFrame(i); CreateFrame(ntp_timestamp_ms, kWidth, kHeight));
WaitForEncodedFrame(ntp_timestamp_ms);
ntp_timestamp_ms += 20;
} }
video_stream_encoder_->Stop(); video_stream_encoder_->Stop();
@ -5516,7 +5532,7 @@ TEST_F(VideoStreamEncoderTest,
// Reconfigure the encoder with a new (higher max framerate), max fps should // Reconfigure the encoder with a new (higher max framerate), max fps should
// still respect the adaptation. // still respect the adaptation.
video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate; video_encoder_config.simulcast_layers[0].max_framerate = kHighFramerate;
source.IncomingCapturedFrame(CreateFrame(1, kFrameWidth, kFrameHeight)); source.IncomingCapturedFrame(CreateFrame(2, kFrameWidth, kFrameHeight));
video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config), video_stream_encoder_->ConfigureEncoder(std::move(video_encoder_config),
kMaxPayloadLength); kMaxPayloadLength);
video_stream_encoder_->WaitUntilTaskQueueIsIdle(); video_stream_encoder_->WaitUntilTaskQueueIsIdle();
@ -9513,7 +9529,6 @@ TEST(VideoStreamEncoderFrameCadenceTest, UsesFrameCadenceAdapterForFrameRate) {
/*max_data_payload_length=*/1000); /*max_data_payload_length=*/1000);
EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps); EXPECT_CALL(*adapter_ptr, GetInputFrameRateFps);
EXPECT_CALL(*adapter_ptr, UpdateFrameRate);
PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1); PassAFrame(encoder_queue, video_stream_encoder_callback, /*ntp_time_ms=*/1);
factory.DepleteTaskQueues(); factory.DepleteTaskQueues();
} }