mirror of
https://github.com/mollyim/webrtc.git
synced 2025-05-13 05:40:42 +01:00
acfd279a14
Implements a two-pass approach to packetization which creates
packets of an even size similar to RtpPacketizer::SplitAboutEqually.
This improves the bandwidth estimation.
The algorithm does a first pass with the existing packetizer, then
iterates through the resulting packet sizes and sums up the bytes left unused in each packet.
It then calculates a new maximum packet length as
configured_max_packet_len - ((unused_bytes - packets + 1) / packets)
adjusts for the overhead and re-runs the packetization algorithm.
For example, a list of OBUs with sizes
{1206, 1476, 1431}
currently gets packetized greedily as payload sizes
{1200, 1200, 1200, 523}
With this change, it gets packetized as
{1032, 1032, 1032, 1028}
This change is guarded by the field trial
WebRTC-Video-AV1EvenPayloadSizes
which is acting as a rollout flag.
BUG=webrtc:15927
Co-authored-by: Shyam Sadhwani <shyamsadhwani@meta.com>
Change-Id: I4f0b3c27de6f06104908dd769c4dd1f34115712c
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/348100
Commit-Queue: Philipp Hancke <phancke@meta.com>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Reviewed-by: Per Kjellander <perkj@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#42203}
376 lines
15 KiB
C++
376 lines
15 KiB
C++
/*
|
|
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "modules/rtp_rtcp/source/rtp_packetizer_av1.h"
|
|
|
|
#include <stddef.h>
|
|
#include <stdint.h>
|
|
|
|
#include <initializer_list>
|
|
#include <utility>
|
|
#include <vector>
|
|
|
|
#include "api/array_view.h"
|
|
#include "api/scoped_refptr.h"
|
|
#include "api/video/encoded_image.h"
|
|
#include "modules/rtp_rtcp/source/rtp_packet_to_send.h"
|
|
#include "modules/rtp_rtcp/source/rtp_packetizer_av1_test_helper.h"
|
|
#include "modules/rtp_rtcp/source/video_rtp_depacketizer_av1.h"
|
|
#include "test/gmock.h"
|
|
#include "test/gtest.h"
|
|
|
|
namespace webrtc {
|
|
namespace {
|
|
|
|
using ::testing::Each;
|
|
using ::testing::ElementsAre;
|
|
using ::testing::ElementsAreArray;
|
|
using ::testing::Le;
|
|
using ::testing::SizeIs;
|
|
|
|
constexpr uint8_t kNewCodedVideoSequenceBit = 0b00'00'1000;
|
|
|
|
// Wrapper around rtp_packet to make it look like container of payload bytes.
|
|
struct RtpPayload {
|
|
using value_type = rtc::ArrayView<const uint8_t>::value_type;
|
|
using const_iterator = rtc::ArrayView<const uint8_t>::const_iterator;
|
|
|
|
RtpPayload() : rtp_packet(/*extensions=*/nullptr) {}
|
|
RtpPayload& operator=(RtpPayload&&) = default;
|
|
RtpPayload(RtpPayload&&) = default;
|
|
|
|
const_iterator begin() const { return rtp_packet.payload().begin(); }
|
|
const_iterator end() const { return rtp_packet.payload().end(); }
|
|
const uint8_t* data() const { return rtp_packet.payload().data(); }
|
|
size_t size() const { return rtp_packet.payload().size(); }
|
|
|
|
uint8_t aggregation_header() const { return rtp_packet.payload()[0]; }
|
|
|
|
RtpPacketToSend rtp_packet;
|
|
};
|
|
|
|
// Wrapper around frame pointer to make it look like container of bytes with
|
|
// nullptr frame look like empty container.
|
|
class Av1Frame {
|
|
public:
|
|
using value_type = uint8_t;
|
|
using const_iterator = const uint8_t*;
|
|
|
|
explicit Av1Frame(rtc::scoped_refptr<EncodedImageBuffer> frame)
|
|
: frame_(std::move(frame)) {}
|
|
|
|
const_iterator begin() const { return frame_ ? frame_->data() : nullptr; }
|
|
const_iterator end() const {
|
|
return frame_ ? (frame_->data() + frame_->size()) : nullptr;
|
|
}
|
|
|
|
private:
|
|
rtc::scoped_refptr<EncodedImageBuffer> frame_;
|
|
};
|
|
|
|
std::vector<RtpPayload> Packetize(
|
|
rtc::ArrayView<const uint8_t> payload,
|
|
RtpPacketizer::PayloadSizeLimits limits,
|
|
bool even_distribution,
|
|
VideoFrameType frame_type = VideoFrameType::kVideoFrameDelta,
|
|
bool is_last_frame_in_picture = true) {
|
|
// Run code under test.
|
|
RtpPacketizerAv1 packetizer(payload, limits, frame_type,
|
|
is_last_frame_in_picture, even_distribution);
|
|
// Convert result into structure that is easier to run expectation against.
|
|
std::vector<RtpPayload> result(packetizer.NumPackets());
|
|
for (RtpPayload& rtp_payload : result) {
|
|
EXPECT_TRUE(packetizer.NextPacket(&rtp_payload.rtp_packet));
|
|
}
|
|
return result;
|
|
}
|
|
|
|
Av1Frame ReassembleFrame(rtc::ArrayView<const RtpPayload> rtp_payloads) {
|
|
std::vector<rtc::ArrayView<const uint8_t>> payloads(rtp_payloads.size());
|
|
for (size_t i = 0; i < rtp_payloads.size(); ++i) {
|
|
payloads[i] = rtp_payloads[i];
|
|
}
|
|
return Av1Frame(VideoRtpDepacketizerAv1().AssembleFrame(payloads));
|
|
}
|
|
|
|
class RtpPacketizerAv1Test : public ::testing::TestWithParam<bool> {};
|
|
|
|
TEST_P(RtpPacketizerAv1Test, EmptyPayload) {
|
|
RtpPacketizer::PayloadSizeLimits limits;
|
|
RtpPacketizerAv1 packetizer({}, limits, VideoFrameType::kVideoFrameKey, true,
|
|
GetParam());
|
|
EXPECT_EQ(packetizer.NumPackets(), 0u);
|
|
}
|
|
|
|
TEST_P(RtpPacketizerAv1Test, PacketizeOneObuWithoutSizeAndExtension) {
|
|
auto kFrame = BuildAv1Frame({Av1Obu(kAv1ObuTypeFrame)
|
|
.WithoutSize()
|
|
.WithPayload({1, 2, 3, 4, 5, 6, 7})});
|
|
EXPECT_THAT(Packetize(kFrame, {}, GetParam()),
|
|
ElementsAre(ElementsAre(0b00'01'0000, // aggregation header
|
|
kAv1ObuTypeFrame, 1, 2, 3, 4, 5, 6, 7)));
|
|
}
|
|
|
|
TEST_P(RtpPacketizerAv1Test, PacketizeOneObuWithoutSizeWithExtension) {
|
|
auto kFrame = BuildAv1Frame({Av1Obu(kAv1ObuTypeFrame)
|
|
.WithoutSize()
|
|
.WithExtension(kAv1ObuExtensionS1T1)
|
|
.WithPayload({2, 3, 4, 5, 6, 7})});
|
|
EXPECT_THAT(
|
|
Packetize(kFrame, {}, GetParam()),
|
|
ElementsAre(ElementsAre(0b00'01'0000, // aggregation header
|
|
kAv1ObuTypeFrame | kAv1ObuExtensionPresentBit,
|
|
kAv1ObuExtensionS1T1, 2, 3, 4, 5, 6, 7)));
|
|
}
|
|
|
|
TEST_P(RtpPacketizerAv1Test, RemovesObuSizeFieldWithoutExtension) {
|
|
auto kFrame = BuildAv1Frame(
|
|
{Av1Obu(kAv1ObuTypeFrame).WithPayload({11, 12, 13, 14, 15, 16, 17})});
|
|
EXPECT_THAT(
|
|
Packetize(kFrame, {}, GetParam()),
|
|
ElementsAre(ElementsAre(0b00'01'0000, // aggregation header
|
|
kAv1ObuTypeFrame, 11, 12, 13, 14, 15, 16, 17)));
|
|
}
|
|
|
|
TEST_P(RtpPacketizerAv1Test, RemovesObuSizeFieldWithExtension) {
|
|
auto kFrame = BuildAv1Frame({Av1Obu(kAv1ObuTypeFrame)
|
|
.WithExtension(kAv1ObuExtensionS1T1)
|
|
.WithPayload({1, 2, 3, 4, 5, 6, 7})});
|
|
EXPECT_THAT(
|
|
Packetize(kFrame, {}, GetParam()),
|
|
ElementsAre(ElementsAre(0b00'01'0000, // aggregation header
|
|
kAv1ObuTypeFrame | kAv1ObuExtensionPresentBit,
|
|
kAv1ObuExtensionS1T1, 1, 2, 3, 4, 5, 6, 7)));
|
|
}
|
|
|
|
TEST_P(RtpPacketizerAv1Test,
|
|
OmitsSizeForLastObuWhenThreeObusFitsIntoThePacket) {
|
|
auto kFrame = BuildAv1Frame(
|
|
{Av1Obu(kAv1ObuTypeSequenceHeader).WithPayload({1, 2, 3, 4, 5, 6}),
|
|
Av1Obu(kAv1ObuTypeMetadata).WithPayload({11, 12, 13, 14}),
|
|
Av1Obu(kAv1ObuTypeFrame).WithPayload({21, 22, 23, 24, 25, 26})});
|
|
EXPECT_THAT(Packetize(kFrame, {}, GetParam()),
|
|
ElementsAre(ElementsAre(
|
|
0b00'11'0000, // aggregation header
|
|
7, kAv1ObuTypeSequenceHeader, 1, 2, 3, 4, 5, 6, //
|
|
5, kAv1ObuTypeMetadata, 11, 12, 13, 14, //
|
|
kAv1ObuTypeFrame, 21, 22, 23, 24, 25, 26)));
|
|
}
|
|
|
|
TEST_P(RtpPacketizerAv1Test, UseSizeForAllObusWhenFourObusFitsIntoThePacket) {
|
|
auto kFrame = BuildAv1Frame(
|
|
{Av1Obu(kAv1ObuTypeSequenceHeader).WithPayload({1, 2, 3, 4, 5, 6}),
|
|
Av1Obu(kAv1ObuTypeMetadata).WithPayload({11, 12, 13, 14}),
|
|
Av1Obu(kAv1ObuTypeFrameHeader).WithPayload({21, 22, 23}),
|
|
Av1Obu(kAv1ObuTypeTileGroup).WithPayload({31, 32, 33, 34, 35, 36})});
|
|
EXPECT_THAT(Packetize(kFrame, {}, GetParam()),
|
|
ElementsAre(ElementsAre(
|
|
0b00'00'0000, // aggregation header
|
|
7, kAv1ObuTypeSequenceHeader, 1, 2, 3, 4, 5, 6, //
|
|
5, kAv1ObuTypeMetadata, 11, 12, 13, 14, //
|
|
4, kAv1ObuTypeFrameHeader, 21, 22, 23, //
|
|
7, kAv1ObuTypeTileGroup, 31, 32, 33, 34, 35, 36)));
|
|
}
|
|
|
|
TEST_P(RtpPacketizerAv1Test, DiscardsTemporalDelimiterAndTileListObu) {
|
|
auto kFrame = BuildAv1Frame(
|
|
{Av1Obu(kAv1ObuTypeTemporalDelimiter), Av1Obu(kAv1ObuTypeMetadata),
|
|
Av1Obu(kAv1ObuTypeTileList).WithPayload({1, 2, 3, 4, 5, 6}),
|
|
Av1Obu(kAv1ObuTypeFrameHeader).WithPayload({21, 22, 23}),
|
|
Av1Obu(kAv1ObuTypeTileGroup).WithPayload({31, 32, 33, 34, 35, 36})});
|
|
|
|
EXPECT_THAT(
|
|
Packetize(kFrame, {}, GetParam()),
|
|
ElementsAre(ElementsAre(0b00'11'0000, // aggregation header
|
|
1,
|
|
kAv1ObuTypeMetadata, //
|
|
4, kAv1ObuTypeFrameHeader, 21, 22,
|
|
23, //
|
|
kAv1ObuTypeTileGroup, 31, 32, 33, 34, 35, 36)));
|
|
}
|
|
|
|
TEST_P(RtpPacketizerAv1Test, SplitTwoObusIntoTwoPacketForceSplitObuHeader) {
|
|
// Craft expected payloads so that there is only one way to split original
|
|
// frame into two packets.
|
|
const uint8_t kExpectPayload1[6] = {
|
|
0b01'10'0000, // aggregation_header
|
|
3,
|
|
kAv1ObuTypeFrameHeader | kAv1ObuExtensionPresentBit,
|
|
kAv1ObuExtensionS1T1,
|
|
21, //
|
|
kAv1ObuTypeTileGroup | kAv1ObuExtensionPresentBit};
|
|
const uint8_t kExpectPayload2[6] = {0b10'01'0000, // aggregation_header
|
|
kAv1ObuExtensionS1T1, 11, 12, 13, 14};
|
|
auto kFrame = BuildAv1Frame({Av1Obu(kAv1ObuTypeFrameHeader)
|
|
.WithExtension(kAv1ObuExtensionS1T1)
|
|
.WithPayload({21}),
|
|
Av1Obu(kAv1ObuTypeTileGroup)
|
|
.WithExtension(kAv1ObuExtensionS1T1)
|
|
.WithPayload({11, 12, 13, 14})});
|
|
|
|
RtpPacketizer::PayloadSizeLimits limits;
|
|
limits.max_payload_len = 6;
|
|
auto payloads = Packetize(kFrame, limits, GetParam());
|
|
EXPECT_THAT(payloads, ElementsAre(ElementsAreArray(kExpectPayload1),
|
|
ElementsAreArray(kExpectPayload2)));
|
|
}
|
|
|
|
TEST_P(RtpPacketizerAv1Test,
|
|
SetsNbitAtTheFirstPacketOfAKeyFrameWithSequenceHeader) {
|
|
auto kFrame = BuildAv1Frame(
|
|
{Av1Obu(kAv1ObuTypeSequenceHeader).WithPayload({1, 2, 3, 4, 5, 6, 7})});
|
|
RtpPacketizer::PayloadSizeLimits limits;
|
|
limits.max_payload_len = 6;
|
|
auto packets =
|
|
Packetize(kFrame, limits, GetParam(), VideoFrameType::kVideoFrameKey);
|
|
ASSERT_THAT(packets, SizeIs(2));
|
|
EXPECT_TRUE(packets[0].aggregation_header() & kNewCodedVideoSequenceBit);
|
|
EXPECT_FALSE(packets[1].aggregation_header() & kNewCodedVideoSequenceBit);
|
|
}
|
|
|
|
TEST_P(RtpPacketizerAv1Test,
|
|
DoesntSetNbitAtThePacketsOfAKeyFrameWithoutSequenceHeader) {
|
|
auto kFrame = BuildAv1Frame(
|
|
{Av1Obu(kAv1ObuTypeFrame).WithPayload({1, 2, 3, 4, 5, 6, 7})});
|
|
RtpPacketizer::PayloadSizeLimits limits;
|
|
limits.max_payload_len = 6;
|
|
auto packets =
|
|
Packetize(kFrame, limits, GetParam(), VideoFrameType::kVideoFrameKey);
|
|
ASSERT_THAT(packets, SizeIs(2));
|
|
EXPECT_FALSE(packets[0].aggregation_header() & kNewCodedVideoSequenceBit);
|
|
EXPECT_FALSE(packets[1].aggregation_header() & kNewCodedVideoSequenceBit);
|
|
}
|
|
|
|
TEST_P(RtpPacketizerAv1Test, DoesntSetNbitAtThePacketsOfADeltaFrame) {
|
|
// Even when that delta frame starts with a (redundant) sequence header.
|
|
auto kFrame = BuildAv1Frame(
|
|
{Av1Obu(kAv1ObuTypeSequenceHeader).WithPayload({1, 2, 3, 4, 5, 6, 7})});
|
|
RtpPacketizer::PayloadSizeLimits limits;
|
|
limits.max_payload_len = 6;
|
|
auto packets =
|
|
Packetize(kFrame, limits, GetParam(), VideoFrameType::kVideoFrameDelta);
|
|
ASSERT_THAT(packets, SizeIs(2));
|
|
EXPECT_FALSE(packets[0].aggregation_header() & kNewCodedVideoSequenceBit);
|
|
EXPECT_FALSE(packets[1].aggregation_header() & kNewCodedVideoSequenceBit);
|
|
}
|
|
|
|
// There are multiple valid reasonable ways to split payload into multiple
|
|
// packets, do not validate current choice, instead use RtpDepacketizer
|
|
// to validate frame is reconstracted to the same one. Note: since
|
|
// RtpDepacketizer always inserts obu_size fields in the output, use frame where
|
|
// each obu has obu_size fields for more streight forward validation.
|
|
TEST_P(RtpPacketizerAv1Test, SplitSingleObuIntoTwoPackets) {
|
|
auto kFrame =
|
|
BuildAv1Frame({Av1Obu(kAv1ObuTypeFrame)
|
|
.WithPayload({11, 12, 13, 14, 15, 16, 17, 18, 19})});
|
|
|
|
RtpPacketizer::PayloadSizeLimits limits;
|
|
limits.max_payload_len = 8;
|
|
auto payloads = Packetize(kFrame, limits, GetParam());
|
|
EXPECT_THAT(payloads, ElementsAre(SizeIs(Le(8u)), SizeIs(Le(8u))));
|
|
|
|
// Use RtpDepacketizer to validate the split.
|
|
EXPECT_THAT(ReassembleFrame(payloads), ElementsAreArray(kFrame));
|
|
}
|
|
|
|
TEST_P(RtpPacketizerAv1Test, SplitSingleObuIntoManyPackets) {
|
|
auto kFrame = BuildAv1Frame(
|
|
{Av1Obu(kAv1ObuTypeFrame).WithPayload(std::vector<uint8_t>(1200, 27))});
|
|
|
|
RtpPacketizer::PayloadSizeLimits limits;
|
|
limits.max_payload_len = 100;
|
|
auto payloads = Packetize(kFrame, limits, GetParam());
|
|
EXPECT_THAT(payloads, SizeIs(13u));
|
|
EXPECT_THAT(payloads, Each(SizeIs(Le(100u))));
|
|
|
|
// Use RtpDepacketizer to validate the split.
|
|
EXPECT_THAT(ReassembleFrame(payloads), ElementsAreArray(kFrame));
|
|
}
|
|
|
|
TEST_P(RtpPacketizerAv1Test, SetMarkerBitForLastPacketInEndOfPictureFrame) {
|
|
auto kFrame = BuildAv1Frame(
|
|
{Av1Obu(kAv1ObuTypeFrame).WithPayload(std::vector<uint8_t>(200, 27))});
|
|
|
|
RtpPacketizer::PayloadSizeLimits limits;
|
|
limits.max_payload_len = 100;
|
|
auto payloads =
|
|
Packetize(kFrame, limits, GetParam(), VideoFrameType::kVideoFrameDelta,
|
|
/*is_last_frame_in_picture=*/true);
|
|
ASSERT_THAT(payloads, SizeIs(3u));
|
|
EXPECT_FALSE(payloads[0].rtp_packet.Marker());
|
|
EXPECT_FALSE(payloads[1].rtp_packet.Marker());
|
|
EXPECT_TRUE(payloads[2].rtp_packet.Marker());
|
|
}
|
|
|
|
TEST_P(RtpPacketizerAv1Test,
|
|
DoesntSetMarkerBitForPacketsNotInEndOfPictureFrame) {
|
|
auto kFrame = BuildAv1Frame(
|
|
{Av1Obu(kAv1ObuTypeFrame).WithPayload(std::vector<uint8_t>(200, 27))});
|
|
|
|
RtpPacketizer::PayloadSizeLimits limits;
|
|
limits.max_payload_len = 100;
|
|
auto payloads =
|
|
Packetize(kFrame, limits, GetParam(), VideoFrameType::kVideoFrameDelta,
|
|
/*is_last_frame_in_picture=*/false);
|
|
ASSERT_THAT(payloads, SizeIs(3u));
|
|
EXPECT_FALSE(payloads[0].rtp_packet.Marker());
|
|
EXPECT_FALSE(payloads[1].rtp_packet.Marker());
|
|
EXPECT_FALSE(payloads[2].rtp_packet.Marker());
|
|
}
|
|
|
|
TEST_P(RtpPacketizerAv1Test, SplitTwoObusIntoTwoPackets) {
|
|
// 2nd OBU is too large to fit into one packet, so its head would be in the
|
|
// same packet as the 1st OBU.
|
|
auto kFrame = BuildAv1Frame(
|
|
{Av1Obu(kAv1ObuTypeSequenceHeader).WithPayload({11, 12}),
|
|
Av1Obu(kAv1ObuTypeFrame).WithPayload({1, 2, 3, 4, 5, 6, 7, 8, 9})});
|
|
|
|
RtpPacketizer::PayloadSizeLimits limits;
|
|
limits.max_payload_len = 8;
|
|
auto payloads = Packetize(kFrame, limits, GetParam());
|
|
EXPECT_THAT(payloads, ElementsAre(SizeIs(Le(8u)), SizeIs(Le(8u))));
|
|
|
|
// Use RtpDepacketizer to validate the split.
|
|
EXPECT_THAT(ReassembleFrame(payloads), ElementsAreArray(kFrame));
|
|
}
|
|
|
|
TEST_P(RtpPacketizerAv1Test,
|
|
SplitSingleObuIntoTwoPacketsBecauseOfSinglePacketLimit) {
|
|
auto kFrame =
|
|
BuildAv1Frame({Av1Obu(kAv1ObuTypeFrame)
|
|
.WithPayload({11, 12, 13, 14, 15, 16, 17, 18, 19})});
|
|
RtpPacketizer::PayloadSizeLimits limits;
|
|
limits.max_payload_len = 10;
|
|
limits.single_packet_reduction_len = 8;
|
|
auto payloads = Packetize(kFrame, limits, GetParam());
|
|
EXPECT_THAT(payloads, ElementsAre(SizeIs(Le(10u)), SizeIs(Le(10u))));
|
|
|
|
EXPECT_THAT(ReassembleFrame(payloads), ElementsAreArray(kFrame));
|
|
}
|
|
|
|
INSTANTIATE_TEST_SUITE_P(bool, RtpPacketizerAv1Test, ::testing::Bool());
|
|
|
|
TEST(RtpPacketizerAv1TestEven, EvenDistributionDiffers) {
|
|
auto kFrame = BuildAv1Frame({
|
|
Av1Obu(kAv1ObuTypeFrame).WithPayload(std::vector<uint8_t>(1206, 0)),
|
|
Av1Obu(kAv1ObuTypeFrame).WithPayload(std::vector<uint8_t>(1476, 0)),
|
|
Av1Obu(kAv1ObuTypeFrame).WithPayload(std::vector<uint8_t>(1431, 0)),
|
|
});
|
|
EXPECT_THAT(
|
|
Packetize(kFrame, {}, /*even_distribution=*/false),
|
|
ElementsAre(SizeIs(1200), SizeIs(1200), SizeIs(1200), SizeIs(523)));
|
|
|
|
EXPECT_THAT(
|
|
Packetize(kFrame, {}, /*even_distribution=*/true),
|
|
ElementsAre(SizeIs(1032), SizeIs(1032), SizeIs(1032), SizeIs(1028)));
|
|
}
|
|
|
|
} // namespace
|
|
} // namespace webrtc
|