mirror of
https://github.com/mollyim/webrtc.git
synced 2025-05-20 00:57:49 +01:00
87b7c1aa6e
There can be error log each frame when maximum playout delay sent with the frame exceed delay derived from the av-sync. In such scenario prefer to limit the playout delay by the one attached to the received frame. Bug: b/390043766 Change-Id: Ia57969df72f7a649e5a9280d5bb29986f5ea14b7 Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/374284 Reviewed-by: Johannes Kron <kron@webrtc.org> Reviewed-by: Ilya Nikolaevskiy <ilnik@webrtc.org> Commit-Queue: Danil Chapovalov <danilchap@webrtc.org> Cr-Commit-Position: refs/heads/main@{#43814}
1287 lines
50 KiB
C++
1287 lines
50 KiB
C++
/*
|
|
* Copyright 2017 The WebRTC Project Authors. All rights reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "video/video_receive_stream2.h"
|
|
|
|
#include <algorithm>
|
|
#include <cstddef>
|
|
#include <deque>
|
|
#include <limits>
|
|
#include <memory>
|
|
#include <optional>
|
|
#include <queue>
|
|
#include <tuple>
|
|
#include <utility>
|
|
#include <vector>
|
|
|
|
#include "absl/memory/memory.h"
|
|
#include "api/environment/environment.h"
|
|
#include "api/environment/environment_factory.h"
|
|
#include "api/metronome/test/fake_metronome.h"
|
|
#include "api/test/mock_video_decoder.h"
|
|
#include "api/test/mock_video_decoder_factory.h"
|
|
#include "api/test/time_controller.h"
|
|
#include "api/units/frequency.h"
|
|
#include "api/units/time_delta.h"
|
|
#include "api/video/encoded_image.h"
|
|
#include "api/video/recordable_encoded_frame.h"
|
|
#include "api/video/test/video_frame_matchers.h"
|
|
#include "api/video/video_frame.h"
|
|
#include "api/video_codecs/sdp_video_format.h"
|
|
#include "api/video_codecs/video_decoder.h"
|
|
#include "call/rtp_stream_receiver_controller.h"
|
|
#include "call/video_receive_stream.h"
|
|
#include "common_video/test/utilities.h"
|
|
#include "media/engine/fake_webrtc_call.h"
|
|
#include "modules/pacing/packet_router.h"
|
|
#include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
|
|
#include "modules/video_coding/encoded_frame.h"
|
|
#include "rtc_base/logging.h"
|
|
#include "system_wrappers/include/clock.h"
|
|
#include "test/fake_decoder.h"
|
|
#include "test/fake_encoded_frame.h"
|
|
#include "test/gmock.h"
|
|
#include "test/gtest.h"
|
|
#include "test/mock_transport.h"
|
|
#include "test/rtcp_packet_parser.h"
|
|
#include "test/time_controller/simulated_time_controller.h"
|
|
#include "test/video_decoder_proxy_factory.h"
|
|
#include "video/call_stats2.h"
|
|
|
|
namespace webrtc {
|
|
|
|
namespace {
|
|
|
|
using test::video_frame_matchers::NtpTimestamp;
|
|
using test::video_frame_matchers::PacketInfos;
|
|
using test::video_frame_matchers::Rotation;
|
|
using ::testing::_;
|
|
using ::testing::AllOf;
|
|
using ::testing::AnyNumber;
|
|
using ::testing::ElementsAreArray;
|
|
using ::testing::Eq;
|
|
using ::testing::Field;
|
|
using ::testing::InSequence;
|
|
using ::testing::Invoke;
|
|
using ::testing::IsEmpty;
|
|
using ::testing::Optional;
|
|
using ::testing::Pointee;
|
|
using ::testing::Property;
|
|
using ::testing::Return;
|
|
using ::testing::SizeIs;
|
|
using ::testing::WithoutArgs;
|
|
|
|
auto RenderedFrameWith(::testing::Matcher<VideoFrame> m) {
|
|
return Optional(m);
|
|
}
|
|
auto RenderedFrame() {
|
|
return RenderedFrameWith(_);
|
|
}
|
|
testing::Matcher<std::optional<VideoFrame>> DidNotReceiveFrame() {
|
|
return Eq(std::nullopt);
|
|
}
|
|
|
|
constexpr TimeDelta kDefaultTimeOut = TimeDelta::Millis(50);
|
|
constexpr int kDefaultNumCpuCores = 2;
|
|
|
|
constexpr Timestamp kStartTime = Timestamp::Millis(1'337'000);
|
|
constexpr Frequency k30Fps = Frequency::Hertz(30);
|
|
constexpr TimeDelta k30FpsDelay = 1 / k30Fps;
|
|
constexpr Frequency kRtpTimestampHz = Frequency::KiloHertz(90);
|
|
constexpr uint32_t k30FpsRtpTimestampDelta = kRtpTimestampHz / k30Fps;
|
|
constexpr uint32_t kFirstRtpTimestamp = 90000;
|
|
constexpr uint8_t kH264PayloadType = 99;
|
|
constexpr uint8_t kH265PayloadType = 100;
|
|
constexpr uint8_t kAv1PayloadType = 101;
|
|
constexpr uint32_t kRemoteSsrc = 1111;
|
|
constexpr uint32_t kLocalSsrc = 2222;
|
|
|
|
class FakeVideoRenderer : public rtc::VideoSinkInterface<VideoFrame> {
|
|
public:
|
|
explicit FakeVideoRenderer(TimeController* time_controller)
|
|
: time_controller_(time_controller) {}
|
|
~FakeVideoRenderer() override = default;
|
|
|
|
void OnFrame(const VideoFrame& frame) override {
|
|
RTC_LOG(LS_VERBOSE) << "Received frame with timestamp="
|
|
<< frame.rtp_timestamp();
|
|
if (!last_frame_.empty()) {
|
|
RTC_LOG(LS_INFO) << "Already had frame queue with timestamp="
|
|
<< last_frame_.back().rtp_timestamp();
|
|
}
|
|
last_frame_.push_back(frame);
|
|
}
|
|
|
|
// If `advance_time`, then the clock will always advance by `timeout`.
|
|
std::optional<VideoFrame> WaitForFrame(TimeDelta timeout,
|
|
bool advance_time = false) {
|
|
auto start = time_controller_->GetClock()->CurrentTime();
|
|
if (last_frame_.empty()) {
|
|
time_controller_->AdvanceTime(TimeDelta::Zero());
|
|
time_controller_->Wait([this] { return !last_frame_.empty(); }, timeout);
|
|
}
|
|
std::optional<VideoFrame> ret;
|
|
if (!last_frame_.empty()) {
|
|
ret = last_frame_.front();
|
|
last_frame_.pop_front();
|
|
}
|
|
if (advance_time) {
|
|
time_controller_->AdvanceTime(
|
|
timeout - (time_controller_->GetClock()->CurrentTime() - start));
|
|
}
|
|
return ret;
|
|
}
|
|
|
|
private:
|
|
std::deque<VideoFrame> last_frame_;
|
|
TimeController* const time_controller_;
|
|
};
|
|
|
|
MATCHER_P2(MatchResolution, w, h, "") {
|
|
return arg.resolution().width == w && arg.resolution().height == h;
|
|
}
|
|
|
|
MATCHER_P(RtpTimestamp, timestamp, "") {
|
|
if (arg.rtp_timestamp() != timestamp) {
|
|
*result_listener->stream()
|
|
<< "rtp timestamp was " << arg.rtp_timestamp() << " != " << timestamp;
|
|
return false;
|
|
}
|
|
return true;
|
|
}
|
|
|
|
// Rtp timestamp for in order frame at 30fps.
|
|
uint32_t RtpTimestampForFrame(int id) {
|
|
return kFirstRtpTimestamp + id * k30FpsRtpTimestampDelta;
|
|
}
|
|
|
|
// Receive time for in order frame at 30fps.
|
|
Timestamp ReceiveTimeForFrame(int id) {
|
|
return kStartTime + id * k30FpsDelay;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
class VideoReceiveStream2Test : public ::testing::TestWithParam<bool> {
|
|
public:
|
|
auto DefaultDecodeAction() {
|
|
return Invoke(&fake_decoder_, &test::FakeDecoder::Decode);
|
|
}
|
|
|
|
bool UseMetronome() const { return GetParam(); }
|
|
|
|
VideoReceiveStream2Test()
|
|
: time_controller_(kStartTime),
|
|
env_(CreateEnvironment(time_controller_.CreateTaskQueueFactory(),
|
|
time_controller_.GetClock())),
|
|
config_(&mock_transport_, &mock_decoder_factory_),
|
|
call_stats_(&env_.clock(), time_controller_.GetMainThread()),
|
|
fake_renderer_(&time_controller_),
|
|
fake_call_(env_),
|
|
fake_metronome_(TimeDelta::Millis(16)),
|
|
decode_sync_(&env_.clock(),
|
|
&fake_metronome_,
|
|
time_controller_.GetMainThread()),
|
|
h264_decoder_factory_(&mock_decoder_) {
|
|
// By default, mock decoder factory is backed by VideoDecoderProxyFactory.
|
|
ON_CALL(mock_decoder_factory_, Create)
|
|
.WillByDefault(Invoke(&h264_decoder_factory_,
|
|
&test::VideoDecoderProxyFactory::Create));
|
|
|
|
// By default, mock decode will wrap the fake decoder.
|
|
ON_CALL(mock_decoder_, Configure)
|
|
.WillByDefault(Invoke(&fake_decoder_, &test::FakeDecoder::Configure));
|
|
ON_CALL(mock_decoder_, Decode(_, _)).WillByDefault(DefaultDecodeAction());
|
|
ON_CALL(mock_decoder_, RegisterDecodeCompleteCallback)
|
|
.WillByDefault(
|
|
Invoke(&fake_decoder_,
|
|
&test::FakeDecoder::RegisterDecodeCompleteCallback));
|
|
ON_CALL(mock_decoder_, Release)
|
|
.WillByDefault(Invoke(&fake_decoder_, &test::FakeDecoder::Release));
|
|
ON_CALL(mock_transport_, SendRtcp)
|
|
.WillByDefault(
|
|
Invoke(&rtcp_packet_parser_, &test::RtcpPacketParser::Parse));
|
|
}
|
|
|
|
~VideoReceiveStream2Test() override {
|
|
if (video_receive_stream_) {
|
|
video_receive_stream_->Stop();
|
|
video_receive_stream_->UnregisterFromTransport();
|
|
}
|
|
time_controller_.AdvanceTime(TimeDelta::Zero());
|
|
}
|
|
|
|
void SetUp() override {
|
|
config_.rtp.remote_ssrc = kRemoteSsrc;
|
|
config_.rtp.local_ssrc = kLocalSsrc;
|
|
config_.renderer = &fake_renderer_;
|
|
VideoReceiveStreamInterface::Decoder h264_decoder;
|
|
h264_decoder.payload_type = kH264PayloadType;
|
|
h264_decoder.video_format = SdpVideoFormat::H264();
|
|
h264_decoder.video_format.parameters.insert(
|
|
{"sprop-parameter-sets", "Z0IACpZTBYmI,aMljiA=="});
|
|
VideoReceiveStreamInterface::Decoder h265_decoder;
|
|
h265_decoder.payload_type = kH265PayloadType;
|
|
h265_decoder.video_format = SdpVideoFormat("H265");
|
|
VideoReceiveStreamInterface::Decoder av1_decoder;
|
|
av1_decoder.payload_type = kAv1PayloadType;
|
|
av1_decoder.video_format = SdpVideoFormat("AV1");
|
|
|
|
config_.decoders = {av1_decoder, h265_decoder, h264_decoder};
|
|
|
|
RecreateReceiveStream();
|
|
}
|
|
|
|
void RecreateReceiveStream(
|
|
std::optional<VideoReceiveStreamInterface::RecordingState> state =
|
|
std::nullopt) {
|
|
if (video_receive_stream_) {
|
|
video_receive_stream_->UnregisterFromTransport();
|
|
video_receive_stream_ = nullptr;
|
|
}
|
|
timing_ = new VCMTiming(&env_.clock(), env_.field_trials());
|
|
video_receive_stream_ =
|
|
std::make_unique<webrtc::internal::VideoReceiveStream2>(
|
|
env_, &fake_call_, kDefaultNumCpuCores, &packet_router_,
|
|
config_.Copy(), &call_stats_, absl::WrapUnique(timing_),
|
|
&nack_periodic_processor_,
|
|
UseMetronome() ? &decode_sync_ : nullptr);
|
|
video_receive_stream_->RegisterWithTransport(
|
|
&rtp_stream_receiver_controller_);
|
|
if (state)
|
|
video_receive_stream_->SetAndGetRecordingState(std::move(*state), false);
|
|
}
|
|
|
|
protected:
|
|
GlobalSimulatedTimeController time_controller_;
|
|
Environment env_;
|
|
NackPeriodicProcessor nack_periodic_processor_;
|
|
testing::NiceMock<MockVideoDecoderFactory> mock_decoder_factory_;
|
|
VideoReceiveStreamInterface::Config config_;
|
|
internal::CallStats call_stats_;
|
|
testing::NiceMock<MockVideoDecoder> mock_decoder_;
|
|
FakeVideoRenderer fake_renderer_;
|
|
cricket::FakeCall fake_call_;
|
|
MockTransport mock_transport_;
|
|
test::RtcpPacketParser rtcp_packet_parser_;
|
|
PacketRouter packet_router_;
|
|
RtpStreamReceiverController rtp_stream_receiver_controller_;
|
|
std::unique_ptr<webrtc::internal::VideoReceiveStream2> video_receive_stream_;
|
|
VCMTiming* timing_;
|
|
test::FakeMetronome fake_metronome_;
|
|
DecodeSynchronizer decode_sync_;
|
|
|
|
private:
|
|
test::VideoDecoderProxyFactory h264_decoder_factory_;
|
|
test::FakeDecoder fake_decoder_;
|
|
};
|
|
|
|
TEST_P(VideoReceiveStream2Test, CreateFrameFromH264FmtpSpropAndIdr) {
|
|
constexpr uint8_t idr_nalu[] = {0x05, 0xFF, 0xFF, 0xFF};
|
|
RtpPacketToSend rtppacket(nullptr);
|
|
uint8_t* payload = rtppacket.AllocatePayload(sizeof(idr_nalu));
|
|
memcpy(payload, idr_nalu, sizeof(idr_nalu));
|
|
rtppacket.SetMarker(true);
|
|
rtppacket.SetSsrc(kRemoteSsrc);
|
|
rtppacket.SetPayloadType(kH264PayloadType);
|
|
rtppacket.SetSequenceNumber(1);
|
|
rtppacket.SetTimestamp(0);
|
|
EXPECT_CALL(mock_decoder_, RegisterDecodeCompleteCallback(_));
|
|
video_receive_stream_->Start();
|
|
EXPECT_CALL(mock_decoder_, Decode(_, _));
|
|
RtpPacketReceived parsed_packet;
|
|
ASSERT_TRUE(parsed_packet.Parse(rtppacket.data(), rtppacket.size()));
|
|
rtp_stream_receiver_controller_.OnRtpPacket(parsed_packet);
|
|
EXPECT_CALL(mock_decoder_, Release());
|
|
|
|
time_controller_.AdvanceTime(TimeDelta::Zero());
|
|
}
|
|
|
|
TEST_P(VideoReceiveStream2Test, PlayoutDelay) {
|
|
const VideoPlayoutDelay kPlayoutDelay(TimeDelta::Millis(123),
|
|
TimeDelta::Millis(521));
|
|
std::unique_ptr<test::FakeEncodedFrame> test_frame =
|
|
test::FakeFrameBuilder()
|
|
.Id(0)
|
|
.PlayoutDelay(kPlayoutDelay)
|
|
.AsLast()
|
|
.Build();
|
|
|
|
video_receive_stream_->OnCompleteFrame(std::move(test_frame));
|
|
auto timings = timing_->GetTimings();
|
|
EXPECT_EQ(kPlayoutDelay.min(), timings.min_playout_delay);
|
|
EXPECT_EQ(kPlayoutDelay.max(), timings.max_playout_delay);
|
|
|
|
// Check that the biggest minimum delay is chosen.
|
|
video_receive_stream_->SetMinimumPlayoutDelay(400);
|
|
timings = timing_->GetTimings();
|
|
EXPECT_EQ(400, timings.min_playout_delay.ms());
|
|
|
|
// Check base minimum delay validation.
|
|
EXPECT_FALSE(video_receive_stream_->SetBaseMinimumPlayoutDelayMs(12345));
|
|
EXPECT_FALSE(video_receive_stream_->SetBaseMinimumPlayoutDelayMs(-1));
|
|
EXPECT_TRUE(video_receive_stream_->SetBaseMinimumPlayoutDelayMs(500));
|
|
timings = timing_->GetTimings();
|
|
EXPECT_EQ(500, timings.min_playout_delay.ms());
|
|
|
|
// Check that intermidiate values are remembered and the biggest remembered
|
|
// is chosen.
|
|
video_receive_stream_->SetBaseMinimumPlayoutDelayMs(0);
|
|
timings = timing_->GetTimings();
|
|
EXPECT_EQ(400, timings.min_playout_delay.ms());
|
|
|
|
video_receive_stream_->SetMinimumPlayoutDelay(0);
|
|
timings = timing_->GetTimings();
|
|
EXPECT_EQ(123, timings.min_playout_delay.ms());
|
|
}
|
|
|
|
TEST_P(VideoReceiveStream2Test, MinPlayoutDelayIsLimitedByMaxPlayoutDelay) {
|
|
const VideoPlayoutDelay kPlayoutDelay(TimeDelta::Millis(123),
|
|
TimeDelta::Millis(321));
|
|
video_receive_stream_->OnCompleteFrame(test::FakeFrameBuilder()
|
|
.Id(0)
|
|
.PlayoutDelay(kPlayoutDelay)
|
|
.AsLast()
|
|
.Build());
|
|
EXPECT_EQ(timing_->GetTimings().min_playout_delay, kPlayoutDelay.min());
|
|
|
|
// Check that the biggest minimum delay is limited by the max playout delay.
|
|
video_receive_stream_->SetMinimumPlayoutDelay(400);
|
|
EXPECT_EQ(timing_->GetTimings().min_playout_delay, kPlayoutDelay.max());
|
|
}
|
|
|
|
TEST_P(VideoReceiveStream2Test, RenderParametersSetToDefaultValues) {
|
|
// Default render parameters.
|
|
const VideoFrame::RenderParameters kDefaultRenderParameters;
|
|
// Default with no playout delay set.
|
|
std::unique_ptr<test::FakeEncodedFrame> test_frame0 =
|
|
test::FakeFrameBuilder().Id(0).AsLast().Build();
|
|
video_receive_stream_->OnCompleteFrame(std::move(test_frame0));
|
|
EXPECT_EQ(timing_->RenderParameters(), kDefaultRenderParameters);
|
|
}
|
|
|
|
TEST_P(VideoReceiveStream2Test, UseLowLatencyRenderingSetFromPlayoutDelay) {
|
|
std::unique_ptr<test::FakeEncodedFrame> test_frame0 =
|
|
test::FakeFrameBuilder()
|
|
.Id(0)
|
|
.PlayoutDelay(VideoPlayoutDelay::Minimal())
|
|
.AsLast()
|
|
.Build();
|
|
video_receive_stream_->OnCompleteFrame(std::move(test_frame0));
|
|
EXPECT_TRUE(timing_->RenderParameters().use_low_latency_rendering);
|
|
|
|
std::unique_ptr<test::FakeEncodedFrame> test_frame1 =
|
|
test::FakeFrameBuilder()
|
|
.Id(1)
|
|
.PlayoutDelay({TimeDelta::Zero(), TimeDelta::Millis(500)})
|
|
.AsLast()
|
|
.Build();
|
|
video_receive_stream_->OnCompleteFrame(std::move(test_frame1));
|
|
EXPECT_TRUE(timing_->RenderParameters().use_low_latency_rendering);
|
|
}
|
|
|
|
TEST_P(VideoReceiveStream2Test, MaxCompositionDelaySetFromMaxPlayoutDelay) {
|
|
// The max composition delay is dependent on the number of frames in the
|
|
// pre-decode queue. It's therefore important to advance the time as the test
|
|
// runs to get the correct expectations of max_composition_delay_in_frames.
|
|
video_receive_stream_->Start();
|
|
// Max composition delay not set if no playout delay is set.
|
|
std::unique_ptr<test::FakeEncodedFrame> test_frame0 =
|
|
test::FakeFrameBuilder()
|
|
.Id(0)
|
|
.Time(RtpTimestampForFrame(0))
|
|
.ReceivedTime(ReceiveTimeForFrame(0))
|
|
.AsLast()
|
|
.Build();
|
|
video_receive_stream_->OnCompleteFrame(std::move(test_frame0));
|
|
EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
|
|
Eq(std::nullopt));
|
|
time_controller_.AdvanceTime(k30FpsDelay);
|
|
|
|
// Max composition delay not set for playout delay 0,0.
|
|
std::unique_ptr<test::FakeEncodedFrame> test_frame1 =
|
|
test::FakeFrameBuilder()
|
|
.Id(1)
|
|
.Time(RtpTimestampForFrame(1))
|
|
.ReceivedTime(ReceiveTimeForFrame(1))
|
|
.PlayoutDelay(VideoPlayoutDelay::Minimal())
|
|
.AsLast()
|
|
.Build();
|
|
video_receive_stream_->OnCompleteFrame(std::move(test_frame1));
|
|
EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
|
|
Eq(std::nullopt));
|
|
time_controller_.AdvanceTime(k30FpsDelay);
|
|
|
|
// Max composition delay not set for playout delay X,Y, where X,Y>0.
|
|
std::unique_ptr<test::FakeEncodedFrame> test_frame2 =
|
|
test::FakeFrameBuilder()
|
|
.Id(2)
|
|
.Time(RtpTimestampForFrame(2))
|
|
.ReceivedTime(ReceiveTimeForFrame(2))
|
|
.PlayoutDelay({TimeDelta::Millis(10), TimeDelta::Millis(30)})
|
|
.AsLast()
|
|
.Build();
|
|
video_receive_stream_->OnCompleteFrame(std::move(test_frame2));
|
|
EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
|
|
Eq(std::nullopt));
|
|
|
|
time_controller_.AdvanceTime(k30FpsDelay);
|
|
|
|
// Max composition delay set if playout delay X,Y, where X=0,Y>0.
|
|
const int kExpectedMaxCompositionDelayInFrames = 3; // ~50 ms at 60 fps.
|
|
std::unique_ptr<test::FakeEncodedFrame> test_frame3 =
|
|
test::FakeFrameBuilder()
|
|
.Id(3)
|
|
.Time(RtpTimestampForFrame(3))
|
|
.ReceivedTime(ReceiveTimeForFrame(3))
|
|
.PlayoutDelay({TimeDelta::Zero(), TimeDelta::Millis(50)})
|
|
.AsLast()
|
|
.Build();
|
|
video_receive_stream_->OnCompleteFrame(std::move(test_frame3));
|
|
EXPECT_THAT(timing_->RenderParameters().max_composition_delay_in_frames,
|
|
Optional(kExpectedMaxCompositionDelayInFrames));
|
|
}
|
|
|
|
TEST_P(VideoReceiveStream2Test, LazyDecoderCreation) {
|
|
constexpr uint8_t idr_nalu[] = {0x05, 0xFF, 0xFF, 0xFF};
|
|
RtpPacketToSend rtppacket(nullptr);
|
|
uint8_t* payload = rtppacket.AllocatePayload(sizeof(idr_nalu));
|
|
memcpy(payload, idr_nalu, sizeof(idr_nalu));
|
|
rtppacket.SetMarker(true);
|
|
rtppacket.SetSsrc(kRemoteSsrc);
|
|
rtppacket.SetPayloadType(kH264PayloadType);
|
|
rtppacket.SetSequenceNumber(1);
|
|
rtppacket.SetTimestamp(0);
|
|
|
|
// No decoders are created by default.
|
|
EXPECT_CALL(mock_decoder_factory_, Create).Times(0);
|
|
video_receive_stream_->Start();
|
|
time_controller_.AdvanceTime(TimeDelta::Zero());
|
|
|
|
EXPECT_TRUE(
|
|
testing::Mock::VerifyAndClearExpectations(&mock_decoder_factory_));
|
|
// Verify that the decoder is created when we receive payload data and tries
|
|
// to decode a frame.
|
|
EXPECT_CALL(mock_decoder_factory_,
|
|
Create(_, Field(&SdpVideoFormat::name, Eq("H264"))));
|
|
EXPECT_CALL(mock_decoder_, Configure);
|
|
EXPECT_CALL(mock_decoder_, RegisterDecodeCompleteCallback);
|
|
EXPECT_CALL(mock_decoder_, Decode(_, _));
|
|
RtpPacketReceived parsed_packet;
|
|
ASSERT_TRUE(parsed_packet.Parse(rtppacket.data(), rtppacket.size()));
|
|
rtp_stream_receiver_controller_.OnRtpPacket(parsed_packet);
|
|
EXPECT_CALL(mock_decoder_, Release);
|
|
|
|
// Make sure the decoder thread had a chance to run.
|
|
time_controller_.AdvanceTime(TimeDelta::Zero());
|
|
}
|
|
|
|
TEST_P(VideoReceiveStream2Test, LazyDecoderCreationCodecSwitch) {
|
|
constexpr uint8_t idr_nalu[] = {0x05, 0xFF, 0xFF, 0xFF};
|
|
RtpPacketToSend rtppacket(nullptr);
|
|
uint8_t* payload = rtppacket.AllocatePayload(sizeof(idr_nalu));
|
|
memcpy(payload, idr_nalu, sizeof(idr_nalu));
|
|
rtppacket.SetMarker(true);
|
|
rtppacket.SetSsrc(kRemoteSsrc);
|
|
rtppacket.SetPayloadType(kH264PayloadType);
|
|
rtppacket.SetSequenceNumber(1);
|
|
rtppacket.SetTimestamp(0);
|
|
|
|
// No decoders are created by default.
|
|
EXPECT_CALL(mock_decoder_factory_, Create).Times(0);
|
|
video_receive_stream_->Start();
|
|
time_controller_.AdvanceTime(TimeDelta::Zero());
|
|
|
|
EXPECT_TRUE(
|
|
testing::Mock::VerifyAndClearExpectations(&mock_decoder_factory_));
|
|
// Verify that the decoder is created when we receive payload data and tries
|
|
// to decode a frame.
|
|
EXPECT_CALL(mock_decoder_factory_,
|
|
Create(_, Field(&SdpVideoFormat::name, Eq("H264"))));
|
|
EXPECT_CALL(mock_decoder_, Configure);
|
|
EXPECT_CALL(mock_decoder_, RegisterDecodeCompleteCallback);
|
|
EXPECT_CALL(mock_decoder_, Decode(_, _));
|
|
RtpPacketReceived parsed_packet;
|
|
ASSERT_TRUE(parsed_packet.Parse(rtppacket.data(), rtppacket.size()));
|
|
rtp_stream_receiver_controller_.OnRtpPacket(parsed_packet);
|
|
// H264 decoder is released after receiving the AV1 packet.
|
|
EXPECT_CALL(mock_decoder_, Release).Times(0);
|
|
|
|
// Make sure the decoder thread had a chance to run.
|
|
time_controller_.AdvanceTime(TimeDelta::Millis(100));
|
|
|
|
// Switch to AV1.
|
|
const uint8_t av1_key_obu[] = {0x18, 0x48, 0x01, 0xAA}; // \ OBU
|
|
RtpPacketToSend av1_rtppacket(nullptr);
|
|
uint8_t* av1_payload = av1_rtppacket.AllocatePayload(sizeof(av1_key_obu));
|
|
memcpy(av1_payload, av1_key_obu, sizeof(av1_key_obu));
|
|
av1_rtppacket.SetMarker(true);
|
|
av1_rtppacket.SetSsrc(kRemoteSsrc);
|
|
av1_rtppacket.SetPayloadType(kAv1PayloadType);
|
|
av1_rtppacket.SetSequenceNumber(2);
|
|
av1_rtppacket.SetTimestamp(1);
|
|
|
|
EXPECT_TRUE(
|
|
testing::Mock::VerifyAndClearExpectations(&mock_decoder_factory_));
|
|
// Release the H264 previous decoder.
|
|
EXPECT_CALL(mock_decoder_, Release);
|
|
// Verify that the decoder is created when we receive payload data and tries
|
|
// to decode a frame.
|
|
EXPECT_CALL(mock_decoder_factory_,
|
|
Create(_, Field(&SdpVideoFormat::name, Eq("AV1"))));
|
|
EXPECT_CALL(mock_decoder_, Configure);
|
|
EXPECT_CALL(mock_decoder_, RegisterDecodeCompleteCallback);
|
|
EXPECT_CALL(mock_decoder_, Decode(_, _));
|
|
ASSERT_TRUE(parsed_packet.Parse(av1_rtppacket.data(), av1_rtppacket.size()));
|
|
rtp_stream_receiver_controller_.OnRtpPacket(parsed_packet);
|
|
|
|
// Make sure the decoder thread had a chance to run.
|
|
time_controller_.AdvanceTime(TimeDelta::Millis(100));
|
|
|
|
// Switch back to H264.
|
|
rtppacket.SetPayloadType(kH264PayloadType);
|
|
rtppacket.SetSequenceNumber(3);
|
|
rtppacket.SetTimestamp(2);
|
|
|
|
EXPECT_TRUE(
|
|
testing::Mock::VerifyAndClearExpectations(&mock_decoder_factory_));
|
|
// Release the AV1 previous decoder and the new H264 decoder on test end.
|
|
EXPECT_CALL(mock_decoder_, Release).Times(2);
|
|
// Verify that the decoder is created when we receive payload data and tries
|
|
// to decode a frame.
|
|
EXPECT_CALL(mock_decoder_factory_,
|
|
Create(_, Field(&SdpVideoFormat::name, Eq("H264"))));
|
|
EXPECT_CALL(mock_decoder_, Configure);
|
|
EXPECT_CALL(mock_decoder_, RegisterDecodeCompleteCallback);
|
|
EXPECT_CALL(mock_decoder_, Decode(_, _));
|
|
ASSERT_TRUE(parsed_packet.Parse(rtppacket.data(), rtppacket.size()));
|
|
rtp_stream_receiver_controller_.OnRtpPacket(parsed_packet);
|
|
|
|
// Make sure the decoder thread had a chance to run.
|
|
time_controller_.AdvanceTime(TimeDelta::Millis(100));
|
|
}
|
|
|
|
TEST_P(VideoReceiveStream2Test, PassesNtpTime) {
|
|
const Timestamp kNtpTimestamp = Timestamp::Millis(12345);
|
|
std::unique_ptr<test::FakeEncodedFrame> test_frame =
|
|
test::FakeFrameBuilder()
|
|
.Id(0)
|
|
.PayloadType(kH264PayloadType)
|
|
.NtpTime(kNtpTimestamp)
|
|
.AsLast()
|
|
.Build();
|
|
|
|
video_receive_stream_->Start();
|
|
video_receive_stream_->OnCompleteFrame(std::move(test_frame));
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
|
|
RenderedFrameWith(NtpTimestamp(kNtpTimestamp)));
|
|
}
|
|
|
|
TEST_P(VideoReceiveStream2Test, PassesRotation) {
|
|
const webrtc::VideoRotation kRotation = webrtc::kVideoRotation_180;
|
|
std::unique_ptr<test::FakeEncodedFrame> test_frame =
|
|
test::FakeFrameBuilder()
|
|
.Id(0)
|
|
.PayloadType(kH264PayloadType)
|
|
.Rotation(kRotation)
|
|
.AsLast()
|
|
.Build();
|
|
|
|
video_receive_stream_->Start();
|
|
video_receive_stream_->OnCompleteFrame(std::move(test_frame));
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
|
|
RenderedFrameWith(Rotation(kRotation)));
|
|
}
|
|
|
|
TEST_P(VideoReceiveStream2Test, PassesPacketInfos) {
|
|
RtpPacketInfos packet_infos = CreatePacketInfos(3);
|
|
auto test_frame = test::FakeFrameBuilder()
|
|
.Id(0)
|
|
.PayloadType(kH264PayloadType)
|
|
.PacketInfos(packet_infos)
|
|
.AsLast()
|
|
.Build();
|
|
|
|
video_receive_stream_->Start();
|
|
video_receive_stream_->OnCompleteFrame(std::move(test_frame));
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
|
|
RenderedFrameWith(PacketInfos(ElementsAreArray(packet_infos))));
|
|
}
|
|
|
|
TEST_P(VideoReceiveStream2Test, RenderedFrameUpdatesGetSources) {
|
|
constexpr uint32_t kSsrc = kRemoteSsrc;
|
|
constexpr uint32_t kCsrc = 9001;
|
|
constexpr uint32_t kRtpTimestamp = 12345;
|
|
|
|
// Prepare one video frame with per-packet information.
|
|
auto test_frame = test::FakeFrameBuilder()
|
|
.Id(0)
|
|
.PayloadType(kH264PayloadType)
|
|
.AsLast()
|
|
.Build();
|
|
RtpPacketInfos packet_infos;
|
|
{
|
|
RtpPacketInfos::vector_type infos;
|
|
|
|
RtpPacketInfo info;
|
|
info.set_ssrc(kSsrc);
|
|
info.set_csrcs({kCsrc});
|
|
info.set_rtp_timestamp(kRtpTimestamp);
|
|
|
|
info.set_receive_time(env_.clock().CurrentTime() - TimeDelta::Millis(5000));
|
|
infos.push_back(info);
|
|
|
|
info.set_receive_time(env_.clock().CurrentTime() - TimeDelta::Millis(3000));
|
|
infos.push_back(info);
|
|
|
|
info.set_receive_time(env_.clock().CurrentTime() - TimeDelta::Millis(2000));
|
|
infos.push_back(info);
|
|
|
|
info.set_receive_time(env_.clock().CurrentTime() - TimeDelta::Millis(1000));
|
|
infos.push_back(info);
|
|
|
|
packet_infos = RtpPacketInfos(std::move(infos));
|
|
}
|
|
test_frame->SetPacketInfos(packet_infos);
|
|
|
|
// Start receive stream.
|
|
video_receive_stream_->Start();
|
|
EXPECT_THAT(video_receive_stream_->GetSources(), IsEmpty());
|
|
|
|
// Render one video frame.
|
|
Timestamp timestamp_min = env_.clock().CurrentTime();
|
|
video_receive_stream_->OnCompleteFrame(std::move(test_frame));
|
|
// Verify that the per-packet information is passed to the renderer.
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut),
|
|
RenderedFrameWith(PacketInfos(ElementsAreArray(packet_infos))));
|
|
Timestamp timestamp_max = env_.clock().CurrentTime();
|
|
|
|
// Verify that the per-packet information also updates `GetSources()`.
|
|
std::vector<RtpSource> sources = video_receive_stream_->GetSources();
|
|
ASSERT_THAT(sources, SizeIs(2));
|
|
{
|
|
auto it = std::find_if(sources.begin(), sources.end(),
|
|
[](const RtpSource& source) {
|
|
return source.source_type() == RtpSourceType::SSRC;
|
|
});
|
|
ASSERT_NE(it, sources.end());
|
|
|
|
EXPECT_EQ(it->source_id(), kSsrc);
|
|
EXPECT_EQ(it->source_type(), RtpSourceType::SSRC);
|
|
EXPECT_EQ(it->rtp_timestamp(), kRtpTimestamp);
|
|
EXPECT_GE(it->timestamp(), timestamp_min);
|
|
EXPECT_LE(it->timestamp(), timestamp_max);
|
|
}
|
|
{
|
|
auto it = std::find_if(sources.begin(), sources.end(),
|
|
[](const RtpSource& source) {
|
|
return source.source_type() == RtpSourceType::CSRC;
|
|
});
|
|
ASSERT_NE(it, sources.end());
|
|
|
|
EXPECT_EQ(it->source_id(), kCsrc);
|
|
EXPECT_EQ(it->source_type(), RtpSourceType::CSRC);
|
|
EXPECT_EQ(it->rtp_timestamp(), kRtpTimestamp);
|
|
EXPECT_GE(it->timestamp(), timestamp_min);
|
|
EXPECT_LE(it->timestamp(), timestamp_max);
|
|
}
|
|
}
|
|
|
|
std::unique_ptr<test::FakeEncodedFrame> MakeFrameWithResolution(
|
|
VideoFrameType frame_type,
|
|
int picture_id,
|
|
int width,
|
|
int height) {
|
|
auto frame = test::FakeFrameBuilder()
|
|
.Id(picture_id)
|
|
.PayloadType(kH264PayloadType)
|
|
.AsLast()
|
|
.Build();
|
|
frame->SetFrameType(frame_type);
|
|
frame->_encodedWidth = width;
|
|
frame->_encodedHeight = height;
|
|
return frame;
|
|
}
|
|
|
|
std::unique_ptr<test::FakeEncodedFrame> MakeFrame(VideoFrameType frame_type,
|
|
int picture_id) {
|
|
return MakeFrameWithResolution(frame_type, picture_id, 320, 240);
|
|
}
|
|
|
|
TEST_P(VideoReceiveStream2Test, PassesFrameWhenEncodedFramesCallbackSet) {
|
|
testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
|
|
video_receive_stream_->Start();
|
|
EXPECT_CALL(callback, Call);
|
|
video_receive_stream_->SetAndGetRecordingState(
|
|
VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
|
|
true);
|
|
video_receive_stream_->OnCompleteFrame(
|
|
MakeFrame(VideoFrameType::kVideoFrameKey, 0));
|
|
EXPECT_TRUE(fake_renderer_.WaitForFrame(kDefaultTimeOut));
|
|
|
|
EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(1));
|
|
|
|
video_receive_stream_->Stop();
|
|
}
|
|
|
|
TEST_P(VideoReceiveStream2Test, MovesEncodedFrameDispatchStateWhenReCreating) {
|
|
testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
|
|
video_receive_stream_->Start();
|
|
// Expect a key frame request over RTCP.
|
|
video_receive_stream_->SetAndGetRecordingState(
|
|
VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
|
|
true);
|
|
video_receive_stream_->Stop();
|
|
VideoReceiveStreamInterface::RecordingState old_state =
|
|
video_receive_stream_->SetAndGetRecordingState(
|
|
VideoReceiveStreamInterface::RecordingState(), false);
|
|
RecreateReceiveStream(std::move(old_state));
|
|
|
|
EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(1));
|
|
|
|
video_receive_stream_->Stop();
|
|
}
|
|
|
|
TEST_P(VideoReceiveStream2Test, RequestsKeyFramesUntilKeyFrameReceived) {
|
|
// Recreate receive stream with shorter delay to test rtx.
|
|
TimeDelta rtx_delay = TimeDelta::Millis(50);
|
|
config_.rtp.nack.rtp_history_ms = rtx_delay.ms();
|
|
auto tick = rtx_delay / 2;
|
|
RecreateReceiveStream();
|
|
video_receive_stream_->Start();
|
|
|
|
video_receive_stream_->GenerateKeyFrame();
|
|
video_receive_stream_->OnCompleteFrame(
|
|
MakeFrame(VideoFrameType::kVideoFrameDelta, 0));
|
|
fake_renderer_.WaitForFrame(kDefaultTimeOut);
|
|
time_controller_.AdvanceTime(tick);
|
|
video_receive_stream_->OnCompleteFrame(
|
|
MakeFrame(VideoFrameType::kVideoFrameDelta, 1));
|
|
fake_renderer_.WaitForFrame(kDefaultTimeOut);
|
|
time_controller_.AdvanceTime(TimeDelta::Zero());
|
|
testing::Mock::VerifyAndClearExpectations(&mock_transport_);
|
|
|
|
EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(1));
|
|
|
|
// T+keyframetimeout: still no key frame received, expect key frame request
|
|
// sent again.
|
|
time_controller_.AdvanceTime(tick);
|
|
video_receive_stream_->OnCompleteFrame(
|
|
MakeFrame(VideoFrameType::kVideoFrameDelta, 2));
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
|
|
testing::Mock::VerifyAndClearExpectations(&mock_transport_);
|
|
|
|
EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(2));
|
|
|
|
// T+keyframetimeout: now send a key frame - we should not observe new key
|
|
// frame requests after this.
|
|
video_receive_stream_->OnCompleteFrame(
|
|
MakeFrame(VideoFrameType::kVideoFrameKey, 3));
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
|
|
time_controller_.AdvanceTime(2 * tick);
|
|
video_receive_stream_->OnCompleteFrame(
|
|
MakeFrame(VideoFrameType::kVideoFrameDelta, 4));
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
|
|
|
|
EXPECT_THAT(rtcp_packet_parser_.pli()->num_packets(), Eq(2));
|
|
}
|
|
|
|
TEST_P(VideoReceiveStream2Test,
|
|
DispatchesEncodedFrameSequenceStartingWithKeyframeWithoutResolution) {
|
|
video_receive_stream_->Start();
|
|
testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
|
|
video_receive_stream_->SetAndGetRecordingState(
|
|
VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
|
|
/*generate_key_frame=*/false);
|
|
|
|
InSequence s;
|
|
EXPECT_CALL(callback,
|
|
Call(MatchResolution(test::FakeDecoder::kDefaultWidth,
|
|
test::FakeDecoder::kDefaultHeight)));
|
|
EXPECT_CALL(callback, Call);
|
|
|
|
video_receive_stream_->OnCompleteFrame(
|
|
MakeFrameWithResolution(VideoFrameType::kVideoFrameKey, 0, 0, 0));
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
|
|
video_receive_stream_->OnCompleteFrame(
|
|
MakeFrameWithResolution(VideoFrameType::kVideoFrameDelta, 1, 0, 0));
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
|
|
|
|
video_receive_stream_->Stop();
|
|
}
|
|
|
|
TEST_P(VideoReceiveStream2Test,
|
|
DispatchesEncodedFrameSequenceStartingWithKeyframeWithResolution) {
|
|
video_receive_stream_->Start();
|
|
testing::MockFunction<void(const RecordableEncodedFrame&)> callback;
|
|
video_receive_stream_->SetAndGetRecordingState(
|
|
VideoReceiveStreamInterface::RecordingState(callback.AsStdFunction()),
|
|
/*generate_key_frame=*/false);
|
|
|
|
InSequence s;
|
|
EXPECT_CALL(callback, Call(MatchResolution(1080u, 720u)));
|
|
EXPECT_CALL(callback, Call);
|
|
|
|
video_receive_stream_->OnCompleteFrame(
|
|
MakeFrameWithResolution(VideoFrameType::kVideoFrameKey, 0, 1080, 720));
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
|
|
video_receive_stream_->OnCompleteFrame(
|
|
MakeFrameWithResolution(VideoFrameType::kVideoFrameDelta, 1, 0, 0));
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(kDefaultTimeOut), RenderedFrame());
|
|
|
|
video_receive_stream_->Stop();
|
|
}
|
|
|
|
TEST_P(VideoReceiveStream2Test, DependantFramesAreScheduled) {
|
|
video_receive_stream_->Start();
|
|
|
|
auto key_frame = test::FakeFrameBuilder()
|
|
.Id(0)
|
|
.PayloadType(kH264PayloadType)
|
|
.Time(kFirstRtpTimestamp)
|
|
.ReceivedTime(kStartTime)
|
|
.AsLast()
|
|
.Build();
|
|
auto delta_frame = test::FakeFrameBuilder()
|
|
.Id(1)
|
|
.PayloadType(kH264PayloadType)
|
|
.Time(RtpTimestampForFrame(1))
|
|
.ReceivedTime(ReceiveTimeForFrame(1))
|
|
.Refs({0})
|
|
.AsLast()
|
|
.Build();
|
|
|
|
// Expect frames are decoded in order.
|
|
InSequence seq;
|
|
EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kFirstRtpTimestamp), _));
|
|
EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kFirstRtpTimestamp +
|
|
k30FpsRtpTimestampDelta),
|
|
_))
|
|
.Times(1);
|
|
video_receive_stream_->OnCompleteFrame(std::move(key_frame));
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
|
|
|
|
time_controller_.AdvanceTime(k30FpsDelay);
|
|
video_receive_stream_->OnCompleteFrame(std::move(delta_frame));
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
|
|
|
|
video_receive_stream_->Stop();
|
|
}
|
|
|
|
TEST_P(VideoReceiveStream2Test, FramesScheduledInOrder) {
|
|
video_receive_stream_->Start();
|
|
|
|
auto key_frame = test::FakeFrameBuilder()
|
|
.Id(0)
|
|
.PayloadType(kH264PayloadType)
|
|
.Time(kFirstRtpTimestamp)
|
|
.AsLast()
|
|
.Build();
|
|
auto delta_frame1 = test::FakeFrameBuilder()
|
|
.Id(1)
|
|
.PayloadType(kH264PayloadType)
|
|
.Time(RtpTimestampForFrame(1))
|
|
.Refs({0})
|
|
.AsLast()
|
|
.Build();
|
|
auto delta_frame2 = test::FakeFrameBuilder()
|
|
.Id(2)
|
|
.PayloadType(kH264PayloadType)
|
|
.Time(RtpTimestampForFrame(2))
|
|
.Refs({1})
|
|
.AsLast()
|
|
.Build();
|
|
|
|
// Expect frames are decoded in order despite delta_frame1 arriving first.
|
|
InSequence seq;
|
|
EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kFirstRtpTimestamp), _))
|
|
.Times(1);
|
|
EXPECT_CALL(mock_decoder_,
|
|
Decode(test::RtpTimestamp(RtpTimestampForFrame(1)), _))
|
|
.Times(1);
|
|
EXPECT_CALL(mock_decoder_,
|
|
Decode(test::RtpTimestamp(RtpTimestampForFrame(2)), _))
|
|
.Times(1);
|
|
key_frame->SetReceivedTime(env_.clock().CurrentTime().ms());
|
|
video_receive_stream_->OnCompleteFrame(std::move(key_frame));
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
|
|
|
|
delta_frame2->SetReceivedTime(env_.clock().CurrentTime().ms());
|
|
video_receive_stream_->OnCompleteFrame(std::move(delta_frame2));
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), DidNotReceiveFrame());
|
|
// `delta_frame1` arrives late.
|
|
delta_frame1->SetReceivedTime(env_.clock().CurrentTime().ms());
|
|
video_receive_stream_->OnCompleteFrame(std::move(delta_frame1));
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay * 2), RenderedFrame());
|
|
video_receive_stream_->Stop();
|
|
}
|
|
|
|
TEST_P(VideoReceiveStream2Test, WaitsforAllSpatialLayers) {
|
|
video_receive_stream_->Start();
|
|
auto sl0 = test::FakeFrameBuilder()
|
|
.Id(0)
|
|
.PayloadType(kH264PayloadType)
|
|
.Time(kFirstRtpTimestamp)
|
|
.ReceivedTime(kStartTime)
|
|
.Build();
|
|
auto sl1 = test::FakeFrameBuilder()
|
|
.Id(1)
|
|
.PayloadType(kH264PayloadType)
|
|
.ReceivedTime(kStartTime)
|
|
.Time(kFirstRtpTimestamp)
|
|
.Refs({0})
|
|
.Build();
|
|
auto sl2 = test::FakeFrameBuilder()
|
|
.Id(2)
|
|
.PayloadType(kH264PayloadType)
|
|
.ReceivedTime(kStartTime)
|
|
.Time(kFirstRtpTimestamp)
|
|
.Refs({0, 1})
|
|
.AsLast()
|
|
.Build();
|
|
|
|
// No decodes should be called until `sl2` is received.
|
|
EXPECT_CALL(mock_decoder_, Decode(_, _)).Times(0);
|
|
sl0->SetReceivedTime(env_.clock().CurrentTime().ms());
|
|
video_receive_stream_->OnCompleteFrame(std::move(sl0));
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
|
|
DidNotReceiveFrame());
|
|
video_receive_stream_->OnCompleteFrame(std::move(sl1));
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
|
|
DidNotReceiveFrame());
|
|
// When `sl2` arrives decode should happen.
|
|
EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kFirstRtpTimestamp), _))
|
|
.Times(1);
|
|
video_receive_stream_->OnCompleteFrame(std::move(sl2));
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
|
|
video_receive_stream_->Stop();
|
|
}
|
|
|
|
TEST_P(VideoReceiveStream2Test, FramesFastForwardOnSystemHalt) {
|
|
video_receive_stream_->Start();
|
|
|
|
// The frame structure looks like this,
|
|
// F1
|
|
// /
|
|
// F0 --> F2
|
|
//
|
|
// In this case we will have a system halt simulated. By the time the system
|
|
// resumes, F1 will be old and so F2 should be decoded.
|
|
auto key_frame = test::FakeFrameBuilder()
|
|
.Id(0)
|
|
.PayloadType(kH264PayloadType)
|
|
.Time(kFirstRtpTimestamp)
|
|
.AsLast()
|
|
.Build();
|
|
auto ffwd_frame = test::FakeFrameBuilder()
|
|
.Id(1)
|
|
.PayloadType(kH264PayloadType)
|
|
.Time(RtpTimestampForFrame(1))
|
|
.Refs({0})
|
|
.AsLast()
|
|
.Build();
|
|
auto rendered_frame = test::FakeFrameBuilder()
|
|
.Id(2)
|
|
.PayloadType(kH264PayloadType)
|
|
.Time(RtpTimestampForFrame(2))
|
|
.Refs({0})
|
|
.AsLast()
|
|
.Build();
|
|
InSequence seq;
|
|
EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kFirstRtpTimestamp), _))
|
|
.WillOnce(testing::DoAll(Invoke([&] {
|
|
// System halt will be simulated in the decode.
|
|
time_controller_.AdvanceTime(k30FpsDelay * 2);
|
|
}),
|
|
DefaultDecodeAction()));
|
|
EXPECT_CALL(mock_decoder_,
|
|
Decode(test::RtpTimestamp(RtpTimestampForFrame(2)), _));
|
|
video_receive_stream_->OnCompleteFrame(std::move(key_frame));
|
|
video_receive_stream_->OnCompleteFrame(std::move(ffwd_frame));
|
|
video_receive_stream_->OnCompleteFrame(std::move(rendered_frame));
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
|
|
RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(0))));
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()),
|
|
RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(2))));
|
|
|
|
// Check stats show correct dropped frames.
|
|
auto stats = video_receive_stream_->GetStats();
|
|
EXPECT_EQ(stats.frames_dropped, 1u);
|
|
|
|
video_receive_stream_->Stop();
|
|
}
|
|
|
|
TEST_P(VideoReceiveStream2Test, BetterFrameInsertedWhileWaitingToDecodeFrame) {
|
|
video_receive_stream_->Start();
|
|
|
|
auto key_frame = test::FakeFrameBuilder()
|
|
.Id(0)
|
|
.PayloadType(kH264PayloadType)
|
|
.Time(kFirstRtpTimestamp)
|
|
.ReceivedTime(ReceiveTimeForFrame(0))
|
|
.AsLast()
|
|
.Build();
|
|
auto f1 = test::FakeFrameBuilder()
|
|
.Id(1)
|
|
.PayloadType(kH264PayloadType)
|
|
.Time(RtpTimestampForFrame(1))
|
|
.ReceivedTime(ReceiveTimeForFrame(1))
|
|
.Refs({0})
|
|
.AsLast()
|
|
.Build();
|
|
auto f2 = test::FakeFrameBuilder()
|
|
.Id(2)
|
|
.PayloadType(kH264PayloadType)
|
|
.Time(RtpTimestampForFrame(2))
|
|
.ReceivedTime(ReceiveTimeForFrame(2))
|
|
.Refs({0})
|
|
.AsLast()
|
|
.Build();
|
|
|
|
video_receive_stream_->OnCompleteFrame(std::move(key_frame));
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
|
|
|
|
InSequence seq;
|
|
EXPECT_CALL(mock_decoder_,
|
|
Decode(test::RtpTimestamp(RtpTimestampForFrame(1)), _))
|
|
.Times(1);
|
|
EXPECT_CALL(mock_decoder_,
|
|
Decode(test::RtpTimestamp(RtpTimestampForFrame(2)), _))
|
|
.Times(1);
|
|
// Simulate f1 arriving after f2 but before f2 is decoded.
|
|
video_receive_stream_->OnCompleteFrame(std::move(f2));
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), DidNotReceiveFrame());
|
|
video_receive_stream_->OnCompleteFrame(std::move(f1));
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
|
|
|
|
video_receive_stream_->Stop();
|
|
}
|
|
|
|
// Note: This test takes a long time (~10s) to run if the fake metronome is
|
|
// active. Since the test needs to wait for the timestamp to rollover, it has a
|
|
// fake delay of around 6.5 hours. Even though time is simulated, this will be
|
|
// around 1,500,000 metronome tick invocations.
|
|
TEST_P(VideoReceiveStream2Test, RtpTimestampWrapAround) {
|
|
EXPECT_CALL(mock_transport_, SendRtcp).Times(AnyNumber());
|
|
video_receive_stream_->Start();
|
|
|
|
constexpr uint32_t kBaseRtp = std::numeric_limits<uint32_t>::max() / 2;
|
|
video_receive_stream_->OnCompleteFrame(
|
|
test::FakeFrameBuilder()
|
|
.Id(0)
|
|
.PayloadType(kH264PayloadType)
|
|
.Time(kBaseRtp)
|
|
.ReceivedTime(env_.clock().CurrentTime())
|
|
.AsLast()
|
|
.Build());
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Zero()), RenderedFrame());
|
|
time_controller_.AdvanceTime(k30FpsDelay);
|
|
video_receive_stream_->OnCompleteFrame(
|
|
test::FakeFrameBuilder()
|
|
.Id(1)
|
|
.PayloadType(kH264PayloadType)
|
|
.Time(kBaseRtp + k30FpsRtpTimestampDelta)
|
|
.ReceivedTime(env_.clock().CurrentTime())
|
|
.AsLast()
|
|
.Build());
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay), RenderedFrame());
|
|
|
|
// Pause stream so that RTP timestamp wraps around.
|
|
constexpr uint32_t kLastRtp = kBaseRtp + k30FpsRtpTimestampDelta;
|
|
constexpr uint32_t kWrapAroundRtp =
|
|
kLastRtp + std::numeric_limits<uint32_t>::max() / 2 + 1;
|
|
// Pause for corresponding delay such that RTP timestamp would increase this
|
|
// much at 30fps.
|
|
constexpr TimeDelta kWrapAroundDelay =
|
|
(std::numeric_limits<uint32_t>::max() / 2 + 1) / kRtpTimestampHz;
|
|
|
|
time_controller_.AdvanceTime(kWrapAroundDelay);
|
|
video_receive_stream_->OnCompleteFrame(
|
|
test::FakeFrameBuilder()
|
|
.Id(2)
|
|
.PayloadType(kH264PayloadType)
|
|
.Time(kWrapAroundRtp)
|
|
.ReceivedTime(env_.clock().CurrentTime())
|
|
.AsLast()
|
|
.Build());
|
|
EXPECT_CALL(mock_decoder_, Decode(test::RtpTimestamp(kWrapAroundRtp), _))
|
|
.Times(1);
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Seconds(1)),
|
|
RenderedFrame());
|
|
|
|
video_receive_stream_->Stop();
|
|
}
|
|
|
|
// If a frame was lost causing the stream to become temporarily non-decodable
|
|
// and the sender reduces their framerate during this time, the video stream
|
|
// should start decoding at the new framerate. However, if the connection is
|
|
// poor, a keyframe will take a long time to send. If the timing of the incoming
|
|
// frames was not kept up to date with the new framerate while the stream was
|
|
// decodable, this late frame will have a large delay as the rtp timestamp of
|
|
// this keyframe will look like the frame arrived early if the frame-rate was
|
|
// not updated.
|
|
TEST_P(VideoReceiveStream2Test, PoorConnectionWithFpsChangeDuringLostFrame) {
|
|
video_receive_stream_->Start();
|
|
|
|
constexpr Frequency k15Fps = Frequency::Hertz(15);
|
|
constexpr TimeDelta k15FpsDelay = 1 / k15Fps;
|
|
constexpr uint32_t k15FpsRtpTimestampDelta = kRtpTimestampHz / k15Fps;
|
|
|
|
// Initial keyframe and frames at 30fps.
|
|
video_receive_stream_->OnCompleteFrame(
|
|
test::FakeFrameBuilder()
|
|
.Id(0)
|
|
.PayloadType(kH264PayloadType)
|
|
.Time(RtpTimestampForFrame(0))
|
|
.ReceivedTime(ReceiveTimeForFrame(0))
|
|
.AsLast()
|
|
.Build());
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
|
|
RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(0))));
|
|
|
|
video_receive_stream_->OnCompleteFrame(
|
|
test::FakeFrameBuilder()
|
|
.Id(1)
|
|
.PayloadType(kH264PayloadType)
|
|
.Time(RtpTimestampForFrame(1))
|
|
.ReceivedTime(ReceiveTimeForFrame(1))
|
|
.Refs({0})
|
|
.AsLast()
|
|
.Build());
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
|
|
RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(1))));
|
|
|
|
// Simulate lost frame 2, followed by 2 second of frames at 30fps, followed by
|
|
// 2 second of frames at 15 fps, and then a keyframe.
|
|
time_controller_.AdvanceTime(k30FpsDelay);
|
|
|
|
Timestamp send_30fps_end_time =
|
|
env_.clock().CurrentTime() + TimeDelta::Seconds(2);
|
|
int id = 3;
|
|
EXPECT_CALL(mock_transport_, SendRtcp).Times(AnyNumber());
|
|
while (env_.clock().CurrentTime() < send_30fps_end_time) {
|
|
++id;
|
|
video_receive_stream_->OnCompleteFrame(
|
|
test::FakeFrameBuilder()
|
|
.Id(id)
|
|
.PayloadType(kH264PayloadType)
|
|
.Time(RtpTimestampForFrame(id))
|
|
.ReceivedTime(ReceiveTimeForFrame(id))
|
|
.Refs({id - 1})
|
|
.AsLast()
|
|
.Build());
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
|
|
Eq(std::nullopt));
|
|
}
|
|
uint32_t current_rtp = RtpTimestampForFrame(id);
|
|
Timestamp send_15fps_end_time =
|
|
env_.clock().CurrentTime() + TimeDelta::Seconds(2);
|
|
while (env_.clock().CurrentTime() < send_15fps_end_time) {
|
|
++id;
|
|
current_rtp += k15FpsRtpTimestampDelta;
|
|
video_receive_stream_->OnCompleteFrame(
|
|
test::FakeFrameBuilder()
|
|
.Id(id)
|
|
.PayloadType(kH264PayloadType)
|
|
.Time(current_rtp)
|
|
.ReceivedTime(env_.clock().CurrentTime())
|
|
.Refs({id - 1})
|
|
.AsLast()
|
|
.Build());
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(k15FpsDelay, /*advance_time=*/true),
|
|
Eq(std::nullopt));
|
|
}
|
|
|
|
++id;
|
|
current_rtp += k15FpsRtpTimestampDelta;
|
|
// Insert keyframe which will recover the stream. However, on a poor
|
|
// connection the keyframe will take significant time to send.
|
|
constexpr TimeDelta kKeyframeDelay = TimeDelta::Millis(200);
|
|
video_receive_stream_->OnCompleteFrame(
|
|
test::FakeFrameBuilder()
|
|
.Id(id)
|
|
.PayloadType(kH264PayloadType)
|
|
.Time(current_rtp)
|
|
.ReceivedTime(env_.clock().CurrentTime() + kKeyframeDelay)
|
|
.AsLast()
|
|
.Build());
|
|
// If the framerate was not updated to be 15fps from the frames that arrived
|
|
// previously, this will fail, as the delay will be longer.
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(k15FpsDelay, /*advance_time=*/true),
|
|
RenderedFrameWith(RtpTimestamp(current_rtp)));
|
|
|
|
video_receive_stream_->Stop();
|
|
}
|
|
|
|
TEST_P(VideoReceiveStream2Test, StreamShouldNotTimeoutWhileWaitingForFrame) {
|
|
// Disable smoothing since this makes it hard to test frame timing.
|
|
config_.enable_prerenderer_smoothing = false;
|
|
RecreateReceiveStream();
|
|
|
|
video_receive_stream_->Start();
|
|
EXPECT_CALL(mock_transport_, SendRtcp).Times(AnyNumber());
|
|
|
|
video_receive_stream_->OnCompleteFrame(
|
|
test::FakeFrameBuilder()
|
|
.Id(0)
|
|
.PayloadType(kH264PayloadType)
|
|
.Time(RtpTimestampForFrame(0))
|
|
.ReceivedTime(ReceiveTimeForFrame(0))
|
|
.AsLast()
|
|
.Build());
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
|
|
RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(0))));
|
|
|
|
for (int id = 1; id < 30; ++id) {
|
|
video_receive_stream_->OnCompleteFrame(
|
|
test::FakeFrameBuilder()
|
|
.Id(id)
|
|
.PayloadType(kH264PayloadType)
|
|
.Time(RtpTimestampForFrame(id))
|
|
.ReceivedTime(ReceiveTimeForFrame(id))
|
|
.Refs({0})
|
|
.AsLast()
|
|
.Build());
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(k30FpsDelay, /*advance_time=*/true),
|
|
RenderedFrameWith(RtpTimestamp(RtpTimestampForFrame(id))));
|
|
}
|
|
|
|
// Simulate a pause in the stream, followed by a decodable frame that is ready
|
|
// long in the future. The stream should not timeout in this case, but rather
|
|
// decode the frame just before the timeout.
|
|
time_controller_.AdvanceTime(TimeDelta::Millis(2900));
|
|
uint32_t late_decode_rtp = kFirstRtpTimestamp + 200 * k30FpsRtpTimestampDelta;
|
|
video_receive_stream_->OnCompleteFrame(
|
|
test::FakeFrameBuilder()
|
|
.Id(121)
|
|
.PayloadType(kH264PayloadType)
|
|
.Time(late_decode_rtp)
|
|
.ReceivedTime(env_.clock().CurrentTime())
|
|
.AsLast()
|
|
.Build());
|
|
EXPECT_THAT(fake_renderer_.WaitForFrame(TimeDelta::Millis(100),
|
|
/*advance_time=*/true),
|
|
RenderedFrameWith(RtpTimestamp(late_decode_rtp)));
|
|
|
|
video_receive_stream_->Stop();
|
|
}
|
|
|
|
INSTANTIATE_TEST_SUITE_P(VideoReceiveStream2Test,
|
|
VideoReceiveStream2Test,
|
|
testing::Bool(),
|
|
[](const auto& test_param_info) {
|
|
return (test_param_info.param
|
|
? "ScheduleDecodesWithMetronome"
|
|
: "ScheduleDecodesWithPostTask");
|
|
});
|
|
|
|
} // namespace webrtc
|