Reland "Fix definition of keyframes decoded statistics"

This is a reland of commit 0e37f5ebd4
with backward compability added to allow downstream tests to migrate to the new signature.

Original change's description:
> Fix definition of keyframes decoded statistics
>
> which are defined to be measured after decoding, not before:
>   https://w3c.github.io/webrtc-stats/#dom-rtcinboundrtpstreamstats-keyframesdecoded
>
> BUG=webrtc:14728
>
> Change-Id: I0a83dde278e1ebe8acf787bdac729af369a1ecf8
> Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/315520
> Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
> Commit-Queue: Philipp Hancke <phancke@microsoft.com>
> Reviewed-by: Henrik Boström <hbos@webrtc.org>
> Cr-Commit-Position: refs/heads/main@{#40545}

BUG=webrtc:14728

Change-Id: I4cf52bb22ba8244155b4fa8c367b9c0306a77590
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/316120
Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org>
Reviewed-by: Henrik Boström <hbos@webrtc.org>
Reviewed-by: Mirko Bonadei <mbonadei@webrtc.org>
Commit-Queue: Philipp Hancke <phancke@microsoft.com>
Cr-Commit-Position: refs/heads/main@{#40553}
This commit is contained in:
Philipp Hancke 2023-08-15 10:20:50 +02:00 committed by WebRTC LUCI CQ
parent 48852c40f4
commit 5165743926
11 changed files with 228 additions and 121 deletions

View file

@ -201,7 +201,8 @@ void VCMDecodedFrameCallback::Decoded(VideoFrame& decodedImage,
decodedImage.set_timestamp_us( decodedImage.set_timestamp_us(
frame_info->render_time ? frame_info->render_time->us() : -1); frame_info->render_time ? frame_info->render_time->us() : -1);
_receiveCallback->FrameToRender(decodedImage, qp, decode_time, _receiveCallback->FrameToRender(decodedImage, qp, decode_time,
frame_info->content_type); frame_info->content_type,
frame_info->frame_type);
} }
void VCMDecodedFrameCallback::OnDecoderInfoChanged( void VCMDecodedFrameCallback::OnDecoderInfoChanged(
@ -285,6 +286,7 @@ int32_t VCMGenericDecoder::Decode(const VCMEncodedFrame& frame, Timestamp now) {
} else { } else {
frame_info.content_type = _last_keyframe_content_type; frame_info.content_type = _last_keyframe_content_type;
} }
frame_info.frame_type = frame.FrameType();
_callback->Map(std::move(frame_info)); _callback->Map(std::move(frame_info));
int32_t ret = decoder_->Decode(frame.EncodedImage(), frame.MissingFrame(), int32_t ret = decoder_->Decode(frame.EncodedImage(), frame.MissingFrame(),

View file

@ -46,6 +46,7 @@ struct FrameInfo {
int64_t ntp_time_ms; int64_t ntp_time_ms;
RtpPacketInfos packet_infos; RtpPacketInfos packet_infos;
// ColorSpace is not stored here, as it might be modified by decoders. // ColorSpace is not stored here, as it might be modified by decoders.
VideoFrameType frame_type;
}; };
class VCMDecodedFrameCallback : public DecodedImageCallback { class VCMDecodedFrameCallback : public DecodedImageCallback {

View file

@ -35,7 +35,8 @@ class ReceiveCallback : public VCMReceiveCallback {
int32_t FrameToRender(VideoFrame& frame, int32_t FrameToRender(VideoFrame& frame,
absl::optional<uint8_t> qp, absl::optional<uint8_t> qp,
TimeDelta decode_time, TimeDelta decode_time,
VideoContentType content_type) override { VideoContentType content_type,
VideoFrameType frame_type) override {
frames_.push_back(frame); frames_.push_back(frame);
return 0; return 0;
} }

View file

@ -34,7 +34,7 @@ namespace webrtc {
enum { enum {
// Timing frames settings. Timing frames are sent every // Timing frames settings. Timing frames are sent every
// `kDefaultTimingFramesDelayMs`, or if the frame is at least // `kDefaultTimingFramesDelayMs`, or if the frame is at least
// `kDefaultOutliserFrameSizePercent` in size of average frame. // `kDefaultOutlierFrameSizePercent` in size of average frame.
kDefaultTimingFramesDelayMs = 200, kDefaultTimingFramesDelayMs = 200,
kDefaultOutlierFrameSizePercent = 500, kDefaultOutlierFrameSizePercent = 500,
// Maximum number of frames for what we store encode start timing information. // Maximum number of frames for what we store encode start timing information.
@ -50,10 +50,22 @@ enum VCMVideoProtection {
// rendered. // rendered.
class VCMReceiveCallback { class VCMReceiveCallback {
public: public:
// TODO(bugs.webrtc.org/14728): make pure virtual again.
virtual int32_t FrameToRender(VideoFrame& videoFrame, // NOLINT virtual int32_t FrameToRender(VideoFrame& videoFrame, // NOLINT
absl::optional<uint8_t> qp, absl::optional<uint8_t> qp,
TimeDelta decode_time, TimeDelta decode_time,
VideoContentType content_type) = 0; VideoContentType content_type,
VideoFrameType frame_type) {
return FrameToRender(videoFrame, qp, decode_time, content_type);
}
// TODO(bugs.webrtc.org/14728): remove this signature.
virtual int32_t FrameToRender(VideoFrame& videoFrame, // NOLINT
absl::optional<uint8_t> qp,
TimeDelta decode_time,
VideoContentType content_type) {
return FrameToRender(videoFrame, qp, decode_time, content_type,
VideoFrameType::kEmptyFrame);
}
virtual void OnDroppedFrames(uint32_t frames_dropped); virtual void OnDroppedFrames(uint32_t frames_dropped);

View file

@ -35,10 +35,13 @@ class MockVCMReceiveCallback : public VCMReceiveCallback {
public: public:
MockVCMReceiveCallback() = default; MockVCMReceiveCallback() = default;
MOCK_METHOD( MOCK_METHOD(int32_t,
int32_t,
FrameToRender, FrameToRender,
(VideoFrame&, absl::optional<uint8_t>, TimeDelta, VideoContentType), (VideoFrame&,
absl::optional<uint8_t>,
TimeDelta,
VideoContentType,
VideoFrameType),
(override)); (override));
MOCK_METHOD(void, OnIncomingPayloadType, (int), (override)); MOCK_METHOD(void, OnIncomingPayloadType, (int), (override));
MOCK_METHOD(void, MOCK_METHOD(void,

View file

@ -38,10 +38,13 @@ class MockVCMReceiveCallback : public VCMReceiveCallback {
MockVCMReceiveCallback() {} MockVCMReceiveCallback() {}
virtual ~MockVCMReceiveCallback() {} virtual ~MockVCMReceiveCallback() {}
MOCK_METHOD( MOCK_METHOD(int32_t,
int32_t,
FrameToRender, FrameToRender,
(VideoFrame&, absl::optional<uint8_t>, TimeDelta, VideoContentType), (VideoFrame&,
absl::optional<uint8_t>,
TimeDelta,
VideoContentType,
VideoFrameType),
(override)); (override));
MOCK_METHOD(void, OnIncomingPayloadType, (int), (override)); MOCK_METHOD(void, OnIncomingPayloadType, (int), (override));
MOCK_METHOD(void, MOCK_METHOD(void,

View file

@ -592,7 +592,8 @@ void ReceiveStatisticsProxy::OnCname(uint32_t ssrc, absl::string_view cname) {
void ReceiveStatisticsProxy::OnDecodedFrame(const VideoFrame& frame, void ReceiveStatisticsProxy::OnDecodedFrame(const VideoFrame& frame,
absl::optional<uint8_t> qp, absl::optional<uint8_t> qp,
TimeDelta decode_time, TimeDelta decode_time,
VideoContentType content_type) { VideoContentType content_type,
VideoFrameType frame_type) {
TimeDelta processing_delay = TimeDelta::Zero(); TimeDelta processing_delay = TimeDelta::Zero();
webrtc::Timestamp current_time = clock_->CurrentTime(); webrtc::Timestamp current_time = clock_->CurrentTime();
// TODO(bugs.webrtc.org/13984): some tests do not fill packet_infos(). // TODO(bugs.webrtc.org/13984): some tests do not fill packet_infos().
@ -615,11 +616,11 @@ void ReceiveStatisticsProxy::OnDecodedFrame(const VideoFrame& frame,
// may be on. E.g. on iOS this gets called on // may be on. E.g. on iOS this gets called on
// "com.apple.coremedia.decompressionsession.clientcallback" // "com.apple.coremedia.decompressionsession.clientcallback"
VideoFrameMetaData meta(frame, current_time); VideoFrameMetaData meta(frame, current_time);
worker_thread_->PostTask( worker_thread_->PostTask(SafeTask(
SafeTask(task_safety_.flag(), [meta, qp, decode_time, processing_delay, task_safety_.flag(), [meta, qp, decode_time, processing_delay,
assembly_time, content_type, this]() { assembly_time, content_type, frame_type, this]() {
OnDecodedFrame(meta, qp, decode_time, processing_delay, assembly_time, OnDecodedFrame(meta, qp, decode_time, processing_delay, assembly_time,
content_type); content_type, frame_type);
})); }));
} }
@ -629,7 +630,8 @@ void ReceiveStatisticsProxy::OnDecodedFrame(
TimeDelta decode_time, TimeDelta decode_time,
TimeDelta processing_delay, TimeDelta processing_delay,
TimeDelta assembly_time, TimeDelta assembly_time,
VideoContentType content_type) { VideoContentType content_type,
VideoFrameType frame_type) {
RTC_DCHECK_RUN_ON(&main_thread_); RTC_DCHECK_RUN_ON(&main_thread_);
const bool is_screenshare = const bool is_screenshare =
@ -651,6 +653,11 @@ void ReceiveStatisticsProxy::OnDecodedFrame(
&content_specific_stats_[content_type]; &content_specific_stats_[content_type];
++stats_.frames_decoded; ++stats_.frames_decoded;
if (frame_type == VideoFrameType::kVideoFrameKey) {
++stats_.frame_counts.key_frames;
} else {
++stats_.frame_counts.delta_frames;
}
if (qp) { if (qp) {
if (!stats_.qp_sum) { if (!stats_.qp_sum) {
if (stats_.frames_decoded != 1) { if (stats_.frames_decoded != 1) {
@ -760,12 +767,6 @@ void ReceiveStatisticsProxy::OnCompleteFrame(bool is_keyframe,
VideoContentType content_type) { VideoContentType content_type) {
RTC_DCHECK_RUN_ON(&main_thread_); RTC_DCHECK_RUN_ON(&main_thread_);
if (is_keyframe) {
++stats_.frame_counts.key_frames;
} else {
++stats_.frame_counts.delta_frames;
}
// Content type extension is set only for keyframes and should be propagated // Content type extension is set only for keyframes and should be propagated
// for all the following delta frames. Here we may receive frames out of order // for all the following delta frames. Here we may receive frames out of order
// and miscategorise some delta frames near the layer switch. // and miscategorise some delta frames near the layer switch.

View file

@ -58,7 +58,8 @@ class ReceiveStatisticsProxy : public VideoStreamBufferControllerStatsObserver,
void OnDecodedFrame(const VideoFrame& frame, void OnDecodedFrame(const VideoFrame& frame,
absl::optional<uint8_t> qp, absl::optional<uint8_t> qp,
TimeDelta decode_time, TimeDelta decode_time,
VideoContentType content_type); VideoContentType content_type,
VideoFrameType frame_type);
// Called asyncronously on the worker thread as a result of a call to the // Called asyncronously on the worker thread as a result of a call to the
// above OnDecodedFrame method, which is called back on the thread where // above OnDecodedFrame method, which is called back on the thread where
@ -68,7 +69,8 @@ class ReceiveStatisticsProxy : public VideoStreamBufferControllerStatsObserver,
TimeDelta decode_time, TimeDelta decode_time,
TimeDelta processing_delay, TimeDelta processing_delay,
TimeDelta assembly_time, TimeDelta assembly_time,
VideoContentType content_type); VideoContentType content_type,
VideoFrameType frame_type);
void OnSyncOffsetUpdated(int64_t video_playout_ntp_ms, void OnSyncOffsetUpdated(int64_t video_playout_ntp_ms,
int64_t sync_offset_ms, int64_t sync_offset_ms,

View file

@ -115,7 +115,8 @@ TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameIncreasesFramesDecoded) {
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
for (uint32_t i = 1; i <= 3; ++i) { for (uint32_t i = 1; i <= 3; ++i) {
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameKey);
EXPECT_EQ(i, FlushAndGetStats().frames_decoded); EXPECT_EQ(i, FlushAndGetStats().frames_decoded);
} }
} }
@ -127,7 +128,8 @@ TEST_F(ReceiveStatisticsProxyTest, DecodedFpsIsReported) {
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
for (int i = 0; i < kRequiredSamples; ++i) { for (int i = 0; i < kRequiredSamples; ++i) {
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameKey);
time_controller_.AdvanceTime(1 / kFps); time_controller_.AdvanceTime(1 / kFps);
} }
FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr); FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr);
@ -144,7 +146,8 @@ TEST_F(ReceiveStatisticsProxyTest, DecodedFpsIsNotReportedForTooFewSamples) {
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
for (int i = 0; i < kRequiredSamples - 1; ++i) { for (int i = 0; i < kRequiredSamples - 1; ++i) {
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameKey);
time_controller_.AdvanceTime(1 / kFps); time_controller_.AdvanceTime(1 / kFps);
} }
FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr); FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr);
@ -159,9 +162,9 @@ TEST_F(ReceiveStatisticsProxyTest,
TimeDelta expected_total_decode_time = TimeDelta::Zero(); TimeDelta expected_total_decode_time = TimeDelta::Zero();
unsigned int expected_frames_decoded = 0; unsigned int expected_frames_decoded = 0;
for (uint32_t i = 1; i <= 3; ++i) { for (uint32_t i = 1; i <= 3; ++i) {
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, statistics_proxy_->OnDecodedFrame(
TimeDelta::Millis(1), frame, absl::nullopt, TimeDelta::Millis(1),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameKey);
expected_total_decode_time += TimeDelta::Millis(1); expected_total_decode_time += TimeDelta::Millis(1);
++expected_frames_decoded; ++expected_frames_decoded;
time_controller_.AdvanceTime(TimeDelta::Zero()); time_controller_.AdvanceTime(TimeDelta::Zero());
@ -171,7 +174,8 @@ TEST_F(ReceiveStatisticsProxyTest,
statistics_proxy_->GetStats().total_decode_time); statistics_proxy_->GetStats().total_decode_time);
} }
statistics_proxy_->OnDecodedFrame(frame, 1u, TimeDelta::Millis(3), statistics_proxy_->OnDecodedFrame(frame, 1u, TimeDelta::Millis(3),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameKey);
++expected_frames_decoded; ++expected_frames_decoded;
expected_total_decode_time += TimeDelta::Millis(3); expected_total_decode_time += TimeDelta::Millis(3);
time_controller_.AdvanceTime(TimeDelta::Zero()); time_controller_.AdvanceTime(TimeDelta::Zero());
@ -195,9 +199,9 @@ TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameIncreasesProcessingDelay) {
frame.set_packet_infos(RtpPacketInfos(packet_infos)); frame.set_packet_infos(RtpPacketInfos(packet_infos));
for (int i = 1; i <= 3; ++i) { for (int i = 1; i <= 3; ++i) {
time_controller_.AdvanceTime(kProcessingDelay); time_controller_.AdvanceTime(kProcessingDelay);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, statistics_proxy_->OnDecodedFrame(
TimeDelta::Millis(1), frame, absl::nullopt, TimeDelta::Millis(1),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameKey);
expected_total_processing_delay += i * kProcessingDelay; expected_total_processing_delay += i * kProcessingDelay;
++expected_frames_decoded; ++expected_frames_decoded;
time_controller_.AdvanceTime(TimeDelta::Zero()); time_controller_.AdvanceTime(TimeDelta::Zero());
@ -208,7 +212,8 @@ TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameIncreasesProcessingDelay) {
} }
time_controller_.AdvanceTime(kProcessingDelay); time_controller_.AdvanceTime(kProcessingDelay);
statistics_proxy_->OnDecodedFrame(frame, 1u, TimeDelta::Millis(3), statistics_proxy_->OnDecodedFrame(frame, 1u, TimeDelta::Millis(3),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameKey);
++expected_frames_decoded; ++expected_frames_decoded;
expected_total_processing_delay += 4 * kProcessingDelay; expected_total_processing_delay += 4 * kProcessingDelay;
time_controller_.AdvanceTime(TimeDelta::Zero()); time_controller_.AdvanceTime(TimeDelta::Zero());
@ -232,7 +237,8 @@ TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameIncreasesAssemblyTime) {
/*ssrc=*/{}, /*csrcs=*/{}, /*rtp_timestamp=*/{}, /*receive_time=*/Now())}; /*ssrc=*/{}, /*csrcs=*/{}, /*rtp_timestamp=*/{}, /*receive_time=*/Now())};
frame.set_packet_infos(RtpPacketInfos(single_packet_frame)); frame.set_packet_infos(RtpPacketInfos(single_packet_frame));
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Millis(1), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Millis(1),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameKey);
++expected_frames_decoded; ++expected_frames_decoded;
time_controller_.AdvanceTime(TimeDelta::Zero()); time_controller_.AdvanceTime(TimeDelta::Zero());
EXPECT_EQ(expected_total_assembly_time, EXPECT_EQ(expected_total_assembly_time,
@ -252,7 +258,8 @@ TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameIncreasesAssemblyTime) {
}; };
frame.set_packet_infos(RtpPacketInfos(ordered_frame)); frame.set_packet_infos(RtpPacketInfos(ordered_frame));
statistics_proxy_->OnDecodedFrame(frame, 1u, TimeDelta::Millis(3), statistics_proxy_->OnDecodedFrame(frame, 1u, TimeDelta::Millis(3),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameKey);
++expected_frames_decoded; ++expected_frames_decoded;
++expected_frames_assembled_from_multiple_packets; ++expected_frames_assembled_from_multiple_packets;
expected_total_assembly_time += 2 * kAssemblyTime; expected_total_assembly_time += 2 * kAssemblyTime;
@ -276,7 +283,8 @@ TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameIncreasesAssemblyTime) {
}; };
frame.set_packet_infos(RtpPacketInfos(unordered_frame)); frame.set_packet_infos(RtpPacketInfos(unordered_frame));
statistics_proxy_->OnDecodedFrame(frame, 1u, TimeDelta::Millis(3), statistics_proxy_->OnDecodedFrame(frame, 1u, TimeDelta::Millis(3),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameKey);
++expected_frames_decoded; ++expected_frames_decoded;
++expected_frames_assembled_from_multiple_packets; ++expected_frames_assembled_from_multiple_packets;
expected_total_assembly_time += 2 * kAssemblyTime; expected_total_assembly_time += 2 * kAssemblyTime;
@ -294,10 +302,12 @@ TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameIncreasesQpSum) {
EXPECT_EQ(absl::nullopt, statistics_proxy_->GetStats().qp_sum); EXPECT_EQ(absl::nullopt, statistics_proxy_->GetStats().qp_sum);
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameKey);
EXPECT_EQ(3u, FlushAndGetStats().qp_sum); EXPECT_EQ(3u, FlushAndGetStats().qp_sum);
statistics_proxy_->OnDecodedFrame(frame, 127u, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, 127u, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameDelta);
EXPECT_EQ(130u, FlushAndGetStats().qp_sum); EXPECT_EQ(130u, FlushAndGetStats().qp_sum);
} }
@ -305,10 +315,12 @@ TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameIncreasesTotalDecodeTime) {
EXPECT_EQ(absl::nullopt, statistics_proxy_->GetStats().qp_sum); EXPECT_EQ(absl::nullopt, statistics_proxy_->GetStats().qp_sum);
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Millis(4), statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Millis(4),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameKey);
EXPECT_EQ(4u, FlushAndGetStats().total_decode_time.ms()); EXPECT_EQ(4u, FlushAndGetStats().total_decode_time.ms());
statistics_proxy_->OnDecodedFrame(frame, 127u, TimeDelta::Millis(7), statistics_proxy_->OnDecodedFrame(frame, 127u, TimeDelta::Millis(7),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameDelta);
EXPECT_EQ(11u, FlushAndGetStats().total_decode_time.ms()); EXPECT_EQ(11u, FlushAndGetStats().total_decode_time.ms());
} }
@ -319,11 +331,13 @@ TEST_F(ReceiveStatisticsProxyTest, ReportsContentType) {
EXPECT_EQ(kRealtimeString, videocontenttypehelpers::ToString( EXPECT_EQ(kRealtimeString, videocontenttypehelpers::ToString(
statistics_proxy_->GetStats().content_type)); statistics_proxy_->GetStats().content_type));
statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Zero(),
VideoContentType::SCREENSHARE); VideoContentType::SCREENSHARE,
VideoFrameType::kVideoFrameKey);
EXPECT_EQ(kScreenshareString, EXPECT_EQ(kScreenshareString,
videocontenttypehelpers::ToString(FlushAndGetStats().content_type)); videocontenttypehelpers::ToString(FlushAndGetStats().content_type));
statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameDelta);
EXPECT_EQ(kRealtimeString, EXPECT_EQ(kRealtimeString,
videocontenttypehelpers::ToString(FlushAndGetStats().content_type)); videocontenttypehelpers::ToString(FlushAndGetStats().content_type));
} }
@ -335,22 +349,26 @@ TEST_F(ReceiveStatisticsProxyTest, ReportsMaxInterframeDelay) {
const TimeDelta kInterframeDelay3 = TimeDelta::Millis(100); const TimeDelta kInterframeDelay3 = TimeDelta::Millis(100);
EXPECT_EQ(-1, statistics_proxy_->GetStats().interframe_delay_max_ms); EXPECT_EQ(-1, statistics_proxy_->GetStats().interframe_delay_max_ms);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameKey);
EXPECT_EQ(-1, FlushAndGetStats().interframe_delay_max_ms); EXPECT_EQ(-1, FlushAndGetStats().interframe_delay_max_ms);
time_controller_.AdvanceTime(kInterframeDelay1); time_controller_.AdvanceTime(kInterframeDelay1);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameDelta);
EXPECT_EQ(kInterframeDelay1.ms(), FlushAndGetStats().interframe_delay_max_ms); EXPECT_EQ(kInterframeDelay1.ms(), FlushAndGetStats().interframe_delay_max_ms);
time_controller_.AdvanceTime(kInterframeDelay2); time_controller_.AdvanceTime(kInterframeDelay2);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameDelta);
EXPECT_EQ(kInterframeDelay2.ms(), FlushAndGetStats().interframe_delay_max_ms); EXPECT_EQ(kInterframeDelay2.ms(), FlushAndGetStats().interframe_delay_max_ms);
time_controller_.AdvanceTime(kInterframeDelay3); time_controller_.AdvanceTime(kInterframeDelay3);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameDelta);
// kInterframeDelay3 is smaller than kInterframeDelay2. // kInterframeDelay3 is smaller than kInterframeDelay2.
EXPECT_EQ(kInterframeDelay2.ms(), FlushAndGetStats().interframe_delay_max_ms); EXPECT_EQ(kInterframeDelay2.ms(), FlushAndGetStats().interframe_delay_max_ms);
} }
@ -362,23 +380,27 @@ TEST_F(ReceiveStatisticsProxyTest, ReportInterframeDelayInWindow) {
const TimeDelta kInterframeDelay3 = TimeDelta::Millis(700); const TimeDelta kInterframeDelay3 = TimeDelta::Millis(700);
EXPECT_EQ(-1, statistics_proxy_->GetStats().interframe_delay_max_ms); EXPECT_EQ(-1, statistics_proxy_->GetStats().interframe_delay_max_ms);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameKey);
EXPECT_EQ(-1, FlushAndGetStats().interframe_delay_max_ms); EXPECT_EQ(-1, FlushAndGetStats().interframe_delay_max_ms);
time_controller_.AdvanceTime(kInterframeDelay1); time_controller_.AdvanceTime(kInterframeDelay1);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameDelta);
EXPECT_EQ(kInterframeDelay1.ms(), FlushAndGetStats().interframe_delay_max_ms); EXPECT_EQ(kInterframeDelay1.ms(), FlushAndGetStats().interframe_delay_max_ms);
time_controller_.AdvanceTime(kInterframeDelay2); time_controller_.AdvanceTime(kInterframeDelay2);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameDelta);
// Still first delay is the maximum // Still first delay is the maximum
EXPECT_EQ(kInterframeDelay1.ms(), FlushAndGetStats().interframe_delay_max_ms); EXPECT_EQ(kInterframeDelay1.ms(), FlushAndGetStats().interframe_delay_max_ms);
time_controller_.AdvanceTime(kInterframeDelay3); time_controller_.AdvanceTime(kInterframeDelay3);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameDelta);
// Now the first sample is out of the window, so the second is the maximum. // Now the first sample is out of the window, so the second is the maximum.
EXPECT_EQ(kInterframeDelay2.ms(), FlushAndGetStats().interframe_delay_max_ms); EXPECT_EQ(kInterframeDelay2.ms(), FlushAndGetStats().interframe_delay_max_ms);
} }
@ -487,7 +509,8 @@ TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameWithoutQpQpSumWontExist) {
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
EXPECT_EQ(absl::nullopt, statistics_proxy_->GetStats().qp_sum); EXPECT_EQ(absl::nullopt, statistics_proxy_->GetStats().qp_sum);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameKey);
EXPECT_EQ(absl::nullopt, FlushAndGetStats().qp_sum); EXPECT_EQ(absl::nullopt, FlushAndGetStats().qp_sum);
} }
@ -495,10 +518,12 @@ TEST_F(ReceiveStatisticsProxyTest, OnDecodedFrameWithoutQpResetsQpSum) {
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
EXPECT_EQ(absl::nullopt, statistics_proxy_->GetStats().qp_sum); EXPECT_EQ(absl::nullopt, statistics_proxy_->GetStats().qp_sum);
statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, 3u, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameKey);
EXPECT_EQ(3u, FlushAndGetStats().qp_sum); EXPECT_EQ(3u, FlushAndGetStats().qp_sum);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameDelta);
EXPECT_EQ(absl::nullopt, FlushAndGetStats().qp_sum); EXPECT_EQ(absl::nullopt, FlushAndGetStats().qp_sum);
} }
@ -543,7 +568,7 @@ TEST_F(ReceiveStatisticsProxyTest, GetStatsReportsOnCompleteFrame) {
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED);
VideoReceiveStreamInterface::Stats stats = statistics_proxy_->GetStats(); VideoReceiveStreamInterface::Stats stats = statistics_proxy_->GetStats();
EXPECT_EQ(1, stats.network_frame_rate); EXPECT_EQ(1, stats.network_frame_rate);
EXPECT_EQ(1, stats.frame_counts.key_frames); EXPECT_EQ(0, stats.frame_counts.key_frames);
EXPECT_EQ(0, stats.frame_counts.delta_frames); EXPECT_EQ(0, stats.frame_counts.delta_frames);
} }
@ -628,16 +653,21 @@ TEST_F(ReceiveStatisticsProxyTest,
TEST_F(ReceiveStatisticsProxyTest, GetStatsReportsFrameCounts) { TEST_F(ReceiveStatisticsProxyTest, GetStatsReportsFrameCounts) {
const int kKeyFrames = 3; const int kKeyFrames = 3;
const int kDeltaFrames = 22; const int kDeltaFrames = 22;
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
for (int i = 0; i < kKeyFrames; i++) { for (int i = 0; i < kKeyFrames; i++) {
statistics_proxy_->OnCompleteFrame(true, 0, VideoContentType::UNSPECIFIED); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameKey);
} }
for (int i = 0; i < kDeltaFrames; i++) { for (int i = 0; i < kDeltaFrames; i++) {
statistics_proxy_->OnCompleteFrame(false, 0, VideoContentType::UNSPECIFIED); statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameDelta);
} }
VideoReceiveStreamInterface::Stats stats = statistics_proxy_->GetStats(); VideoReceiveStreamInterface::Stats stats = statistics_proxy_->GetStats();
EXPECT_EQ(kKeyFrames, stats.frame_counts.key_frames); EXPECT_EQ(0, stats.frame_counts.key_frames);
EXPECT_EQ(kDeltaFrames, stats.frame_counts.delta_frames); EXPECT_EQ(0, stats.frame_counts.delta_frames);
} }
TEST_F(ReceiveStatisticsProxyTest, GetStatsReportsCName) { TEST_F(ReceiveStatisticsProxyTest, GetStatsReportsCName) {
@ -700,8 +730,15 @@ TEST_F(ReceiveStatisticsProxyTest, RespectsReportingIntervalForTimingFrames) {
TEST_F(ReceiveStatisticsProxyTest, LifetimeHistogramIsUpdated) { TEST_F(ReceiveStatisticsProxyTest, LifetimeHistogramIsUpdated) {
const TimeDelta kLifetime = TimeDelta::Seconds(3); const TimeDelta kLifetime = TimeDelta::Seconds(3);
time_controller_.AdvanceTime(kLifetime); time_controller_.AdvanceTime(kLifetime);
// Need at least one frame to report stream lifetime. // Need at least one decoded frame to report stream lifetime.
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
statistics_proxy_->OnCompleteFrame(true, 1000, VideoContentType::UNSPECIFIED); statistics_proxy_->OnCompleteFrame(true, 1000, VideoContentType::UNSPECIFIED);
statistics_proxy_->OnDecodedFrame(
frame, absl::nullopt, TimeDelta::Millis(1000),
VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameKey);
FlushAndGetStats();
statistics_proxy_->UpdateHistograms(absl::nullopt, StreamDataCounters(), statistics_proxy_->UpdateHistograms(absl::nullopt, StreamDataCounters(),
nullptr); nullptr);
EXPECT_METRIC_EQ( EXPECT_METRIC_EQ(
@ -841,10 +878,16 @@ TEST_F(ReceiveStatisticsProxyTest,
KeyFrameHistogramNotUpdatedForTooFewSamples) { KeyFrameHistogramNotUpdatedForTooFewSamples) {
const bool kIsKeyFrame = false; const bool kIsKeyFrame = false;
const int kFrameSizeBytes = 1000; const int kFrameSizeBytes = 1000;
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
for (int i = 0; i < kMinRequiredSamples - 1; ++i) for (int i = 0; i < kMinRequiredSamples - 1; ++i) {
statistics_proxy_->OnCompleteFrame(kIsKeyFrame, kFrameSizeBytes, statistics_proxy_->OnCompleteFrame(kIsKeyFrame, kFrameSizeBytes,
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED);
statistics_proxy_->OnDecodedFrame(
frame, absl::nullopt, TimeDelta::Millis(1000),
VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameDelta);
}
FlushAndGetStats();
EXPECT_EQ(0, statistics_proxy_->GetStats().frame_counts.key_frames); EXPECT_EQ(0, statistics_proxy_->GetStats().frame_counts.key_frames);
EXPECT_EQ(kMinRequiredSamples - 1, EXPECT_EQ(kMinRequiredSamples - 1,
@ -860,10 +903,16 @@ TEST_F(ReceiveStatisticsProxyTest,
KeyFrameHistogramUpdatedForMinRequiredSamples) { KeyFrameHistogramUpdatedForMinRequiredSamples) {
const bool kIsKeyFrame = false; const bool kIsKeyFrame = false;
const int kFrameSizeBytes = 1000; const int kFrameSizeBytes = 1000;
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
for (int i = 0; i < kMinRequiredSamples; ++i) for (int i = 0; i < kMinRequiredSamples; ++i) {
statistics_proxy_->OnCompleteFrame(kIsKeyFrame, kFrameSizeBytes, statistics_proxy_->OnCompleteFrame(kIsKeyFrame, kFrameSizeBytes,
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED);
statistics_proxy_->OnDecodedFrame(
frame, absl::nullopt, TimeDelta::Millis(1000),
VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameDelta);
}
FlushAndGetStats();
EXPECT_EQ(0, statistics_proxy_->GetStats().frame_counts.key_frames); EXPECT_EQ(0, statistics_proxy_->GetStats().frame_counts.key_frames);
EXPECT_EQ(kMinRequiredSamples, EXPECT_EQ(kMinRequiredSamples,
@ -879,14 +928,23 @@ TEST_F(ReceiveStatisticsProxyTest,
TEST_F(ReceiveStatisticsProxyTest, KeyFrameHistogramIsUpdated) { TEST_F(ReceiveStatisticsProxyTest, KeyFrameHistogramIsUpdated) {
const int kFrameSizeBytes = 1000; const int kFrameSizeBytes = 1000;
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
for (int i = 0; i < kMinRequiredSamples; ++i) for (int i = 0; i < kMinRequiredSamples; ++i) {
statistics_proxy_->OnCompleteFrame(true, kFrameSizeBytes, statistics_proxy_->OnCompleteFrame(true, kFrameSizeBytes,
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED);
statistics_proxy_->OnDecodedFrame(
for (int i = 0; i < kMinRequiredSamples; ++i) frame, absl::nullopt, TimeDelta::Millis(1000),
VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameKey);
}
for (int i = 0; i < kMinRequiredSamples; ++i) {
statistics_proxy_->OnCompleteFrame(false, kFrameSizeBytes, statistics_proxy_->OnCompleteFrame(false, kFrameSizeBytes,
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED);
statistics_proxy_->OnDecodedFrame(
frame, absl::nullopt, TimeDelta::Millis(1000),
VideoContentType::UNSPECIFIED, VideoFrameType::kVideoFrameDelta);
}
FlushAndGetStats();
EXPECT_EQ(kMinRequiredSamples, EXPECT_EQ(kMinRequiredSamples,
statistics_proxy_->GetStats().frame_counts.key_frames); statistics_proxy_->GetStats().frame_counts.key_frames);
@ -966,7 +1024,8 @@ TEST_F(ReceiveStatisticsProxyTest, DoesNotReportStaleFramerates) {
frame.set_ntp_time_ms( frame.set_ntp_time_ms(
time_controller_.GetClock()->CurrentNtpInMilliseconds()); time_controller_.GetClock()->CurrentNtpInMilliseconds());
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameKey);
statistics_proxy_->OnRenderedFrame(MetaData(frame)); statistics_proxy_->OnRenderedFrame(MetaData(frame));
time_controller_.AdvanceTime(1 / kDefaultFps); time_controller_.AdvanceTime(1 / kDefaultFps);
} }
@ -1038,7 +1097,8 @@ TEST_F(ReceiveStatisticsProxyTest, ReceivedFrameHistogramsAreUpdated) {
TEST_F(ReceiveStatisticsProxyTest, ZeroDelayReportedIfFrameNotDelayed) { TEST_F(ReceiveStatisticsProxyTest, ZeroDelayReportedIfFrameNotDelayed) {
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameKey);
// Frame not delayed, delayed frames to render: 0%. // Frame not delayed, delayed frames to render: 0%.
statistics_proxy_->OnRenderedFrame( statistics_proxy_->OnRenderedFrame(
@ -1060,7 +1120,8 @@ TEST_F(ReceiveStatisticsProxyTest,
DelayedFrameHistogramsAreNotUpdatedIfMinRuntimeHasNotPassed) { DelayedFrameHistogramsAreNotUpdatedIfMinRuntimeHasNotPassed) {
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameKey);
// Frame not delayed, delayed frames to render: 0%. // Frame not delayed, delayed frames to render: 0%.
statistics_proxy_->OnRenderedFrame( statistics_proxy_->OnRenderedFrame(
@ -1081,7 +1142,8 @@ TEST_F(ReceiveStatisticsProxyTest,
DelayedFramesHistogramsAreNotUpdatedIfNoRenderedFrames) { DelayedFramesHistogramsAreNotUpdatedIfNoRenderedFrames) {
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameKey);
// Min run time has passed. No rendered frames. // Min run time has passed. No rendered frames.
time_controller_.AdvanceTime( time_controller_.AdvanceTime(
@ -1097,7 +1159,8 @@ TEST_F(ReceiveStatisticsProxyTest,
TEST_F(ReceiveStatisticsProxyTest, DelayReportedIfFrameIsDelayed) { TEST_F(ReceiveStatisticsProxyTest, DelayReportedIfFrameIsDelayed) {
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameKey);
// Frame delayed 1 ms, delayed frames to render: 100%. // Frame delayed 1 ms, delayed frames to render: 100%.
statistics_proxy_->OnRenderedFrame( statistics_proxy_->OnRenderedFrame(
@ -1121,7 +1184,8 @@ TEST_F(ReceiveStatisticsProxyTest, DelayReportedIfFrameIsDelayed) {
TEST_F(ReceiveStatisticsProxyTest, AverageDelayOfDelayedFramesIsReported) { TEST_F(ReceiveStatisticsProxyTest, AverageDelayOfDelayedFramesIsReported) {
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
VideoContentType::UNSPECIFIED); VideoContentType::UNSPECIFIED,
VideoFrameType::kVideoFrameKey);
// Two frames delayed (6 ms, 10 ms), delayed frames to render: 50%. // Two frames delayed (6 ms, 10 ms), delayed frames to render: 50%.
const int64_t kNowMs = Now().ms(); const int64_t kNowMs = Now().ms();
@ -1274,13 +1338,15 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, InterFrameDelaysAreReported) {
for (int i = 0; i < kMinRequiredSamples; ++i) { for (int i = 0; i < kMinRequiredSamples; ++i) {
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
content_type_); content_type_,
VideoFrameType::kVideoFrameKey);
time_controller_.AdvanceTime(kInterFrameDelay); time_controller_.AdvanceTime(kInterFrameDelay);
} }
// One extra with double the interval. // One extra with double the interval.
time_controller_.AdvanceTime(kInterFrameDelay); time_controller_.AdvanceTime(kInterFrameDelay);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
content_type_); content_type_,
VideoFrameType::kVideoFrameDelta);
FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr); FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr);
const TimeDelta kExpectedInterFrame = const TimeDelta kExpectedInterFrame =
@ -1310,18 +1376,21 @@ TEST_P(ReceiveStatisticsProxyTestWithContent,
for (int i = 0; i <= kMinRequiredSamples - kLastFivePercentsSamples; ++i) { for (int i = 0; i <= kMinRequiredSamples - kLastFivePercentsSamples; ++i) {
time_controller_.AdvanceTime(kInterFrameDelay); time_controller_.AdvanceTime(kInterFrameDelay);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
content_type_); content_type_,
VideoFrameType::kVideoFrameKey);
} }
// Last 5% of intervals are double in size. // Last 5% of intervals are double in size.
for (int i = 0; i < kLastFivePercentsSamples; ++i) { for (int i = 0; i < kLastFivePercentsSamples; ++i) {
time_controller_.AdvanceTime(2 * kInterFrameDelay); time_controller_.AdvanceTime(2 * kInterFrameDelay);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
content_type_); content_type_,
VideoFrameType::kVideoFrameKey);
} }
// Final sample is outlier and 10 times as big. // Final sample is outlier and 10 times as big.
time_controller_.AdvanceTime(10 * kInterFrameDelay); time_controller_.AdvanceTime(10 * kInterFrameDelay);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
content_type_); content_type_,
VideoFrameType::kVideoFrameKey);
FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr); FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr);
const TimeDelta kExpectedInterFrame = kInterFrameDelay * 2; const TimeDelta kExpectedInterFrame = kInterFrameDelay * 2;
@ -1344,7 +1413,8 @@ TEST_P(ReceiveStatisticsProxyTestWithContent,
for (int i = 0; i < kMinRequiredSamples; ++i) { for (int i = 0; i < kMinRequiredSamples; ++i) {
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
content_type_); content_type_,
VideoFrameType::kVideoFrameKey);
time_controller_.AdvanceTime(kInterFrameDelay); time_controller_.AdvanceTime(kInterFrameDelay);
} }
@ -1367,7 +1437,8 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, MaxInterFrameDelayOnlyWithPause) {
for (int i = 0; i <= kMinRequiredSamples; ++i) { for (int i = 0; i <= kMinRequiredSamples; ++i) {
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
content_type_); content_type_,
VideoFrameType::kVideoFrameKey);
time_controller_.AdvanceTime(kInterFrameDelay); time_controller_.AdvanceTime(kInterFrameDelay);
} }
@ -1378,10 +1449,12 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, MaxInterFrameDelayOnlyWithPause) {
// Insert two more frames. The interval during the pause should be // Insert two more frames. The interval during the pause should be
// disregarded in the stats. // disregarded in the stats.
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
content_type_); content_type_,
VideoFrameType::kVideoFrameKey);
time_controller_.AdvanceTime(kInterFrameDelay); time_controller_.AdvanceTime(kInterFrameDelay);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
content_type_); content_type_,
VideoFrameType::kVideoFrameDelta);
FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr); FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr);
if (videocontenttypehelpers::IsScreenshare(content_type_)) { if (videocontenttypehelpers::IsScreenshare(content_type_)) {
@ -1416,18 +1489,18 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, FreezesAreReported) {
for (int i = 0; i < kMinRequiredSamples; ++i) { for (int i = 0; i < kMinRequiredSamples; ++i) {
VideoFrameMetaData meta = MetaData(frame); VideoFrameMetaData meta = MetaData(frame);
statistics_proxy_->OnDecodedFrame(meta, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(
TimeDelta::Zero(), TimeDelta::Zero(), meta, absl::nullopt, TimeDelta::Zero(), TimeDelta::Zero(),
content_type_); TimeDelta::Zero(), content_type_, VideoFrameType::kVideoFrameKey);
statistics_proxy_->OnRenderedFrame(meta); statistics_proxy_->OnRenderedFrame(meta);
time_controller_.AdvanceTime(kInterFrameDelay); time_controller_.AdvanceTime(kInterFrameDelay);
} }
// Add extra freeze. // Add extra freeze.
time_controller_.AdvanceTime(kFreezeDelay); time_controller_.AdvanceTime(kFreezeDelay);
VideoFrameMetaData meta = MetaData(frame); VideoFrameMetaData meta = MetaData(frame);
statistics_proxy_->OnDecodedFrame(meta, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(
TimeDelta::Zero(), TimeDelta::Zero(), meta, absl::nullopt, TimeDelta::Zero(), TimeDelta::Zero(),
content_type_); TimeDelta::Zero(), content_type_, VideoFrameType::kVideoFrameDelta);
statistics_proxy_->OnRenderedFrame(meta); statistics_proxy_->OnRenderedFrame(meta);
FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr); FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr);
@ -1466,7 +1539,8 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, HarmonicFrameRateIsReported) {
for (int i = 0; i < kMinRequiredSamples; ++i) { for (int i = 0; i < kMinRequiredSamples; ++i) {
time_controller_.AdvanceTime(kFrameDuration); time_controller_.AdvanceTime(kFrameDuration);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
content_type_); content_type_,
VideoFrameType::kVideoFrameKey);
statistics_proxy_->OnRenderedFrame(MetaData(frame)); statistics_proxy_->OnRenderedFrame(MetaData(frame));
} }
@ -1474,14 +1548,16 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, HarmonicFrameRateIsReported) {
// Add freeze. // Add freeze.
time_controller_.AdvanceTime(kFreezeDuration); time_controller_.AdvanceTime(kFreezeDuration);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
content_type_); content_type_,
VideoFrameType::kVideoFrameDelta);
statistics_proxy_->OnRenderedFrame(MetaData(frame)); statistics_proxy_->OnRenderedFrame(MetaData(frame));
// Add pause. // Add pause.
time_controller_.AdvanceTime(kPauseDuration); time_controller_.AdvanceTime(kPauseDuration);
statistics_proxy_->OnStreamInactive(); statistics_proxy_->OnStreamInactive();
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
content_type_); content_type_,
VideoFrameType::kVideoFrameDelta);
statistics_proxy_->OnRenderedFrame(MetaData(frame)); statistics_proxy_->OnRenderedFrame(MetaData(frame));
FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr); FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr);
@ -1511,9 +1587,9 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, PausesAreIgnored) {
for (int i = 0; i <= kMinRequiredSamples; ++i) { for (int i = 0; i <= kMinRequiredSamples; ++i) {
VideoFrameMetaData meta = MetaData(frame); VideoFrameMetaData meta = MetaData(frame);
statistics_proxy_->OnDecodedFrame(meta, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(
TimeDelta::Zero(), TimeDelta::Zero(), meta, absl::nullopt, TimeDelta::Zero(), TimeDelta::Zero(),
content_type_); TimeDelta::Zero(), content_type_, VideoFrameType::kVideoFrameKey);
statistics_proxy_->OnRenderedFrame(meta); statistics_proxy_->OnRenderedFrame(meta);
time_controller_.AdvanceTime(kInterFrameDelay); time_controller_.AdvanceTime(kInterFrameDelay);
} }
@ -1523,9 +1599,9 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, PausesAreIgnored) {
// Second playback interval with triple the length. // Second playback interval with triple the length.
for (int i = 0; i <= kMinRequiredSamples * 3; ++i) { for (int i = 0; i <= kMinRequiredSamples * 3; ++i) {
VideoFrameMetaData meta = MetaData(frame); VideoFrameMetaData meta = MetaData(frame);
statistics_proxy_->OnDecodedFrame(meta, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(
TimeDelta::Zero(), TimeDelta::Zero(), meta, absl::nullopt, TimeDelta::Zero(), TimeDelta::Zero(),
content_type_); TimeDelta::Zero(), content_type_, VideoFrameType::kVideoFrameDelta);
statistics_proxy_->OnRenderedFrame(meta); statistics_proxy_->OnRenderedFrame(meta);
time_controller_.AdvanceTime(kInterFrameDelay); time_controller_.AdvanceTime(kInterFrameDelay);
} }
@ -1556,12 +1632,14 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, ManyPausesAtTheBeginning) {
for (int i = 0; i <= kMinRequiredSamples; ++i) { for (int i = 0; i <= kMinRequiredSamples; ++i) {
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
content_type_); content_type_,
VideoFrameType::kVideoFrameKey);
time_controller_.AdvanceTime(kInterFrameDelay); time_controller_.AdvanceTime(kInterFrameDelay);
statistics_proxy_->OnStreamInactive(); statistics_proxy_->OnStreamInactive();
time_controller_.AdvanceTime(kPauseDuration); time_controller_.AdvanceTime(kPauseDuration);
statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, absl::nullopt, TimeDelta::Zero(),
content_type_); content_type_,
VideoFrameType::kVideoFrameDelta);
time_controller_.AdvanceTime(kInterFrameDelay); time_controller_.AdvanceTime(kInterFrameDelay);
} }
@ -1586,18 +1664,18 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, TimeInHdReported) {
// HD frames. // HD frames.
for (int i = 0; i < kMinRequiredSamples; ++i) { for (int i = 0; i < kMinRequiredSamples; ++i) {
VideoFrameMetaData meta = MetaData(frame_hd); VideoFrameMetaData meta = MetaData(frame_hd);
statistics_proxy_->OnDecodedFrame(meta, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(
TimeDelta::Zero(), TimeDelta::Zero(), meta, absl::nullopt, TimeDelta::Zero(), TimeDelta::Zero(),
content_type_); TimeDelta::Zero(), content_type_, VideoFrameType::kVideoFrameKey);
statistics_proxy_->OnRenderedFrame(meta); statistics_proxy_->OnRenderedFrame(meta);
time_controller_.AdvanceTime(kInterFrameDelay); time_controller_.AdvanceTime(kInterFrameDelay);
} }
// SD frames. // SD frames.
for (int i = 0; i < 2 * kMinRequiredSamples; ++i) { for (int i = 0; i < 2 * kMinRequiredSamples; ++i) {
VideoFrameMetaData meta = MetaData(frame_sd); VideoFrameMetaData meta = MetaData(frame_sd);
statistics_proxy_->OnDecodedFrame(meta, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(
TimeDelta::Zero(), TimeDelta::Zero(), meta, absl::nullopt, TimeDelta::Zero(), TimeDelta::Zero(),
content_type_); TimeDelta::Zero(), content_type_, VideoFrameType::kVideoFrameKey);
statistics_proxy_->OnRenderedFrame(meta); statistics_proxy_->OnRenderedFrame(meta);
time_controller_.AdvanceTime(kInterFrameDelay); time_controller_.AdvanceTime(kInterFrameDelay);
} }
@ -1626,24 +1704,25 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, TimeInBlockyVideoReported) {
// High quality frames. // High quality frames.
for (int i = 0; i < kMinRequiredSamples; ++i) { for (int i = 0; i < kMinRequiredSamples; ++i) {
VideoFrameMetaData meta = MetaData(frame); VideoFrameMetaData meta = MetaData(frame);
statistics_proxy_->OnDecodedFrame(meta, kLowQp, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(
TimeDelta::Zero(), TimeDelta::Zero(), meta, kLowQp, TimeDelta::Zero(), TimeDelta::Zero(), TimeDelta::Zero(),
content_type_); content_type_, VideoFrameType::kVideoFrameKey);
statistics_proxy_->OnRenderedFrame(meta); statistics_proxy_->OnRenderedFrame(meta);
time_controller_.AdvanceTime(kInterFrameDelay); time_controller_.AdvanceTime(kInterFrameDelay);
} }
// Blocky frames. // Blocky frames.
for (int i = 0; i < 2 * kMinRequiredSamples; ++i) { for (int i = 0; i < 2 * kMinRequiredSamples; ++i) {
VideoFrameMetaData meta = MetaData(frame); VideoFrameMetaData meta = MetaData(frame);
statistics_proxy_->OnDecodedFrame(meta, kHighQp, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(
TimeDelta::Zero(), TimeDelta::Zero(), meta, kHighQp, TimeDelta::Zero(), TimeDelta::Zero(), TimeDelta::Zero(),
content_type_); content_type_, VideoFrameType::kVideoFrameKey);
statistics_proxy_->OnRenderedFrame(meta); statistics_proxy_->OnRenderedFrame(meta);
time_controller_.AdvanceTime(kInterFrameDelay); time_controller_.AdvanceTime(kInterFrameDelay);
} }
// Extra last frame. // Extra last frame.
statistics_proxy_->OnDecodedFrame(frame, kHighQp, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame, kHighQp, TimeDelta::Zero(),
content_type_); content_type_,
VideoFrameType::kVideoFrameKey);
statistics_proxy_->OnRenderedFrame(MetaData(frame)); statistics_proxy_->OnRenderedFrame(MetaData(frame));
FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr); FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr);
@ -1670,7 +1749,8 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, DownscalesReported) {
// Call once to pass content type. // Call once to pass content type.
statistics_proxy_->OnDecodedFrame(frame_hd, absl::nullopt, TimeDelta::Zero(), statistics_proxy_->OnDecodedFrame(frame_hd, absl::nullopt, TimeDelta::Zero(),
content_type_); content_type_,
VideoFrameType::kVideoFrameKey);
time_controller_.AdvanceTime(TimeDelta::Zero()); time_controller_.AdvanceTime(TimeDelta::Zero());
statistics_proxy_->OnRenderedFrame(MetaData(frame_hd)); statistics_proxy_->OnRenderedFrame(MetaData(frame_hd));
@ -1699,8 +1779,8 @@ TEST_P(ReceiveStatisticsProxyTestWithContent, DecodeTimeReported) {
webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight); webrtc::VideoFrame frame = CreateFrame(kWidth, kHeight);
for (int i = 0; i < kMinRequiredSamples; ++i) { for (int i = 0; i < kMinRequiredSamples; ++i) {
statistics_proxy_->OnDecodedFrame(frame, kLowQp, kDecodeTime, statistics_proxy_->OnDecodedFrame(frame, kLowQp, kDecodeTime, content_type_,
content_type_); VideoFrameType::kVideoFrameKey);
time_controller_.AdvanceTime(kInterFrameDelay); time_controller_.AdvanceTime(kInterFrameDelay);
} }
FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr); FlushAndUpdateHistograms(absl::nullopt, StreamDataCounters(), nullptr);

View file

@ -46,9 +46,10 @@ VideoStreamDecoder::~VideoStreamDecoder() {
int32_t VideoStreamDecoder::FrameToRender(VideoFrame& video_frame, int32_t VideoStreamDecoder::FrameToRender(VideoFrame& video_frame,
absl::optional<uint8_t> qp, absl::optional<uint8_t> qp,
TimeDelta decode_time, TimeDelta decode_time,
VideoContentType content_type) { VideoContentType content_type,
VideoFrameType frame_type) {
receive_stats_callback_->OnDecodedFrame(video_frame, qp, decode_time, receive_stats_callback_->OnDecodedFrame(video_frame, qp, decode_time,
content_type); content_type, frame_type);
incoming_video_stream_->OnFrame(video_frame); incoming_video_stream_->OnFrame(video_frame);
return 0; return 0;
} }

View file

@ -43,7 +43,8 @@ class VideoStreamDecoder : public VCMReceiveCallback {
int32_t FrameToRender(VideoFrame& video_frame, int32_t FrameToRender(VideoFrame& video_frame,
absl::optional<uint8_t> qp, absl::optional<uint8_t> qp,
TimeDelta decode_time, TimeDelta decode_time,
VideoContentType content_type) override; VideoContentType content_type,
VideoFrameType frame_type) override;
void OnDroppedFrames(uint32_t frames_dropped) override; void OnDroppedFrames(uint32_t frames_dropped) override;
void OnIncomingPayloadType(int payload_type) override; void OnIncomingPayloadType(int payload_type) override;
void OnDecoderInfoChanged( void OnDecoderInfoChanged(