From d97b6499c3ef7712ccba48cae32192290e622a5a Mon Sep 17 00:00:00 2001 From: Jianjun Zhu Date: Wed, 3 Apr 2024 15:57:34 +0800 Subject: [PATCH] H26xPacketBuffer handles out of band H.264 parameter sets. This CL updates H26xPacketBuffer to store and prepend SPS and PPS for H.264 bitstreams when IDR only keyframe is allowed. Bug: webrtc:13485 Change-Id: Ic1edc623dff568d54d3ce29b42dd8eab3312f5cb Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/342225 Reviewed-by: Philip Eliasson Commit-Queue: Philip Eliasson Reviewed-by: Sergey Silkin Cr-Commit-Position: refs/heads/main@{#41986} --- modules/video_coding/BUILD.gn | 16 +- modules/video_coding/h26x_packet_buffer.cc | 330 ++++++++++++++---- modules/video_coding/h26x_packet_buffer.h | 54 ++- .../h26x_packet_buffer_unittest.cc | 170 ++++++++- video/BUILD.gn | 1 + 5 files changed, 479 insertions(+), 92 deletions(-) diff --git a/modules/video_coding/BUILD.gn b/modules/video_coding/BUILD.gn index 4da259e215..ef63d2d8bc 100644 --- a/modules/video_coding/BUILD.gn +++ b/modules/video_coding/BUILD.gn @@ -130,6 +130,7 @@ rtc_library("h26x_packet_buffer") { ] deps = [ ":codec_globals_headers", + ":h264_sprop_parameter_sets", ":packet_buffer", "../../api:array_view", "../../api:rtp_packet_info", @@ -162,6 +163,17 @@ rtc_library("frame_helpers") { absl_deps = [ "//third_party/abseil-cpp/absl/container:inlined_vector" ] } +rtc_library("h264_sprop_parameter_sets") { + sources = [ + "h264_sprop_parameter_sets.cc", + "h264_sprop_parameter_sets.h", + ] + deps = [ + "../../rtc_base:logging", + "../../rtc_base/third_party/base64", + ] +} + rtc_library("video_coding") { visibility = [ "*" ] sources = [ @@ -172,8 +184,6 @@ rtc_library("video_coding") { "fec_rate_table.h", "generic_decoder.cc", "generic_decoder.h", - "h264_sprop_parameter_sets.cc", - "h264_sprop_parameter_sets.h", "h264_sps_pps_tracker.cc", "h264_sps_pps_tracker.h", "include/video_codec_initializer.h", @@ -203,6 +213,7 @@ rtc_library("video_coding") { ":codec_globals_headers", ":encoded_frame", ":frame_helpers", + ":h264_sprop_parameter_sets", ":video_codec_interface", ":video_coding_utility", ":webrtc_vp8_scalability", @@ -1173,6 +1184,7 @@ if (rtc_include_tests) { ":encoded_frame", ":frame_dependencies_calculator", ":frame_helpers", + ":h264_sprop_parameter_sets", ":h26x_packet_buffer", ":nack_requester", ":packet_buffer", diff --git a/modules/video_coding/h26x_packet_buffer.cc b/modules/video_coding/h26x_packet_buffer.cc index bca2b5ce29..133ab4708a 100644 --- a/modules/video_coding/h26x_packet_buffer.cc +++ b/modules/video_coding/h26x_packet_buffer.cc @@ -19,10 +19,13 @@ #include "api/rtp_packet_info.h" #include "api/video/video_frame_type.h" #include "common_video/h264/h264_common.h" +#include "common_video/h264/pps_parser.h" +#include "common_video/h264/sps_parser.h" #include "modules/rtp_rtcp/source/rtp_header_extensions.h" #include "modules/rtp_rtcp/source/rtp_packet_received.h" #include "modules/rtp_rtcp/source/rtp_video_header.h" #include "modules/video_coding/codecs/h264/include/h264_globals.h" +#include "modules/video_coding/h264_sprop_parameter_sets.h" #include "rtc_base/checks.h" #include "rtc_base/copy_on_write_buffer.h" #include "rtc_base/logging.h" @@ -91,55 +94,6 @@ bool HasVps(const H26xPacketBuffer::Packet& packet) { } #endif -// TODO(bugs.webrtc.org/13157): Update the H264 depacketizer so we don't have to -// fiddle with the payload at this point. -rtc::CopyOnWriteBuffer FixH264VideoPayload( - rtc::ArrayView payload, - const RTPVideoHeader& video_header) { - constexpr uint8_t kStartCode[] = {0, 0, 0, 1}; - - const auto& h264_header = - absl::get(video_header.video_type_header); - - rtc::CopyOnWriteBuffer result; - switch (h264_header.packetization_type) { - case kH264StapA: { - const uint8_t* payload_end = payload.data() + payload.size(); - const uint8_t* nalu_ptr = payload.data() + 1; - while (nalu_ptr < payload_end - 1) { - // The first two bytes describe the length of the segment, where a - // segment is the nalu type plus nalu payload. - uint16_t segment_length = nalu_ptr[0] << 8 | nalu_ptr[1]; - nalu_ptr += 2; - - if (nalu_ptr + segment_length <= payload_end) { - result.AppendData(kStartCode); - result.AppendData(nalu_ptr, segment_length); - } - nalu_ptr += segment_length; - } - return result; - } - - case kH264FuA: { - if (IsFirstPacketOfFragment(h264_header)) { - result.AppendData(kStartCode); - } - result.AppendData(payload); - return result; - } - - case kH264SingleNalu: { - result.AppendData(kStartCode); - result.AppendData(payload); - return result; - } - } - - RTC_DCHECK_NOTREACHED(); - return result; -} - } // namespace H26xPacketBuffer::H26xPacketBuffer(bool h264_idr_only_keyframes_allowed) @@ -162,8 +116,7 @@ H26xPacketBuffer::InsertResult H26xPacketBuffer::InsertPacket( packet_slot = std::move(packet); } - result.packets = FindFrames(unwrapped_seq_num); - return result; + return FindFrames(unwrapped_seq_num); } std::unique_ptr& H26xPacketBuffer::GetPacket( @@ -185,9 +138,9 @@ bool H26xPacketBuffer::BeginningOfStream( return false; } -std::vector> -H26xPacketBuffer::FindFrames(int64_t unwrapped_seq_num) { - std::vector> found_frames; +H26xPacketBuffer::InsertResult H26xPacketBuffer::FindFrames( + int64_t unwrapped_seq_num) { + InsertResult result; Packet* packet = GetPacket(unwrapped_seq_num).get(); RTC_CHECK(packet != nullptr); @@ -197,7 +150,7 @@ H26xPacketBuffer::FindFrames(int64_t unwrapped_seq_num) { if (unwrapped_seq_num - 1 != last_continuous_unwrapped_seq_num_) { if (unwrapped_seq_num <= last_continuous_unwrapped_seq_num_ || !BeginningOfStream(*packet)) { - return found_frames; + return result; } last_continuous_unwrapped_seq_num_ = unwrapped_seq_num; @@ -211,7 +164,7 @@ H26xPacketBuffer::FindFrames(int64_t unwrapped_seq_num) { // the 'buffer_'. Check that the `packet` sequence number match the expected // unwrapped sequence number. if (static_cast(seq_num) != packet->seq_num) { - return found_frames; + return result; } last_continuous_unwrapped_seq_num_ = seq_num; @@ -225,12 +178,12 @@ H26xPacketBuffer::FindFrames(int64_t unwrapped_seq_num) { auto& prev_packet = GetPacket(seq_num_start - 1); if (prev_packet == nullptr || prev_packet->timestamp != rtp_timestamp) { - if (MaybeAssembleFrame(seq_num_start, seq_num, found_frames)) { + if (MaybeAssembleFrame(seq_num_start, seq_num, result)) { // Frame was assembled, continue to look for more frames. break; } else { // Frame was not assembled, no subsequent frame will be continuous. - return found_frames; + return result; } } } @@ -239,24 +192,23 @@ H26xPacketBuffer::FindFrames(int64_t unwrapped_seq_num) { seq_num++; packet = GetPacket(seq_num).get(); if (packet == nullptr) { - return found_frames; + return result; } } - return found_frames; + return result; } -bool H26xPacketBuffer::MaybeAssembleFrame( - int64_t start_seq_num_unwrapped, - int64_t end_sequence_number_unwrapped, - std::vector>& frames) { +bool H26xPacketBuffer::MaybeAssembleFrame(int64_t start_seq_num_unwrapped, + int64_t end_sequence_number_unwrapped, + InsertResult& result) { #ifdef RTC_ENABLE_H265 bool has_vps = false; #endif bool has_sps = false; bool has_pps = false; + // Includes IDR, CRA and BLA for HEVC. bool has_idr = false; - bool has_irap = false; int width = -1; int height = -1; @@ -284,13 +236,13 @@ bool H26xPacketBuffer::MaybeAssembleFrame( for (const auto& nalu_index : nalu_indices) { uint8_t nalu_type = H265::ParseNaluType( packet->video_payload.cdata()[nalu_index.payload_start_offset]); - has_irap |= (nalu_type >= H265::NaluType::kBlaWLp && - nalu_type <= H265::NaluType::kRsvIrapVcl23); + has_idr |= (nalu_type >= H265::NaluType::kBlaWLp && + nalu_type <= H265::NaluType::kRsvIrapVcl23); has_vps |= nalu_type == H265::NaluType::kVps; has_sps |= nalu_type == H265::NaluType::kSps; has_pps |= nalu_type == H265::NaluType::kPps; } - if (has_irap) { + if (has_idr) { if (!has_vps || !has_sps || !has_pps) { return false; } @@ -317,21 +269,253 @@ bool H26xPacketBuffer::MaybeAssembleFrame( packet->video_header.height = height; } - packet->video_header.frame_type = has_idr || has_irap + packet->video_header.frame_type = has_idr ? VideoFrameType::kVideoFrameKey : VideoFrameType::kVideoFrameDelta; } - // Start code is inserted by depacktizer for H.265. + // Only applies to H.264 because start code is inserted by depacktizer for + // H.265 and out-of-band parameter sets is not supported by H.265. if (packet->codec() == kVideoCodecH264) { - packet->video_payload = - FixH264VideoPayload(packet->video_payload, packet->video_header); + if (!FixH264Packet(*packet)) { + // The buffer is not cleared actually, but a key frame request is + // needed. + result.buffer_cleared = true; + return false; + } } - frames.push_back(std::move(packet)); + result.packets.push_back(std::move(packet)); } return true; } +void H26xPacketBuffer::SetSpropParameterSets( + const std::string& sprop_parameter_sets) { + if (!h264_idr_only_keyframes_allowed_) { + RTC_LOG(LS_WARNING) << "Ignore sprop parameter sets because IDR only " + "keyframe is not allowed."; + return; + } + H264SpropParameterSets sprop_decoder; + if (!sprop_decoder.DecodeSprop(sprop_parameter_sets)) { + return; + } + InsertSpsPpsNalus(sprop_decoder.sps_nalu(), sprop_decoder.pps_nalu()); +} + +void H26xPacketBuffer::InsertSpsPpsNalus(const std::vector& sps, + const std::vector& pps) { + RTC_CHECK(h264_idr_only_keyframes_allowed_); + constexpr size_t kNaluHeaderOffset = 1; + if (sps.size() < kNaluHeaderOffset) { + RTC_LOG(LS_WARNING) << "SPS size " << sps.size() << " is smaller than " + << kNaluHeaderOffset; + return; + } + if ((sps[0] & 0x1f) != H264::NaluType::kSps) { + RTC_LOG(LS_WARNING) << "SPS Nalu header missing"; + return; + } + if (pps.size() < kNaluHeaderOffset) { + RTC_LOG(LS_WARNING) << "PPS size " << pps.size() << " is smaller than " + << kNaluHeaderOffset; + return; + } + if ((pps[0] & 0x1f) != H264::NaluType::kPps) { + RTC_LOG(LS_WARNING) << "SPS Nalu header missing"; + return; + } + absl::optional parsed_sps = SpsParser::ParseSps( + sps.data() + kNaluHeaderOffset, sps.size() - kNaluHeaderOffset); + absl::optional parsed_pps = PpsParser::ParsePps( + pps.data() + kNaluHeaderOffset, pps.size() - kNaluHeaderOffset); + + if (!parsed_sps) { + RTC_LOG(LS_WARNING) << "Failed to parse SPS."; + } + + if (!parsed_pps) { + RTC_LOG(LS_WARNING) << "Failed to parse PPS."; + } + + if (!parsed_pps || !parsed_sps) { + return; + } + + SpsInfo sps_info; + sps_info.size = sps.size(); + sps_info.width = parsed_sps->width; + sps_info.height = parsed_sps->height; + uint8_t* sps_data = new uint8_t[sps_info.size]; + memcpy(sps_data, sps.data(), sps_info.size); + sps_info.payload.reset(sps_data); + sps_data_[parsed_sps->id] = std::move(sps_info); + + PpsInfo pps_info; + pps_info.size = pps.size(); + pps_info.sps_id = parsed_pps->sps_id; + uint8_t* pps_data = new uint8_t[pps_info.size]; + memcpy(pps_data, pps.data(), pps_info.size); + pps_info.payload.reset(pps_data); + pps_data_[parsed_pps->id] = std::move(pps_info); + + RTC_LOG(LS_INFO) << "Inserted SPS id " << parsed_sps->id << " and PPS id " + << parsed_pps->id << " (referencing SPS " + << parsed_pps->sps_id << ")"; +} + +// TODO(bugs.webrtc.org/13157): Update the H264 depacketizer so we don't have to +// fiddle with the payload at this point. +bool H26xPacketBuffer::FixH264Packet(Packet& packet) { + constexpr uint8_t kStartCode[] = {0, 0, 0, 1}; + + RTPVideoHeader& video_header = packet.video_header; + RTPVideoHeaderH264& h264_header = + absl::get(video_header.video_type_header); + + rtc::CopyOnWriteBuffer result; + + if (h264_idr_only_keyframes_allowed_) { + // Check if sps and pps insertion is needed. + bool prepend_sps_pps = false; + auto sps = sps_data_.end(); + auto pps = pps_data_.end(); + + for (size_t i = 0; i < h264_header.nalus_length; ++i) { + const NaluInfo& nalu = h264_header.nalus[i]; + switch (nalu.type) { + case H264::NaluType::kSps: { + SpsInfo& sps_info = sps_data_[nalu.sps_id]; + sps_info.width = video_header.width; + sps_info.height = video_header.height; + break; + } + case H264::NaluType::kPps: { + pps_data_[nalu.pps_id].sps_id = nalu.sps_id; + break; + } + case H264::NaluType::kIdr: { + // If this is the first packet of an IDR, make sure we have the + // required SPS/PPS and also calculate how much extra space we need + // in the buffer to prepend the SPS/PPS to the bitstream with start + // codes. + if (video_header.is_first_packet_in_frame) { + if (nalu.pps_id == -1) { + RTC_LOG(LS_WARNING) << "No PPS id in IDR nalu."; + return false; + } + + pps = pps_data_.find(nalu.pps_id); + if (pps == pps_data_.end()) { + RTC_LOG(LS_WARNING) + << "No PPS with id << " << nalu.pps_id << " received"; + return false; + } + + sps = sps_data_.find(pps->second.sps_id); + if (sps == sps_data_.end()) { + RTC_LOG(LS_WARNING) + << "No SPS with id << " << pps->second.sps_id << " received"; + return false; + } + + // Since the first packet of every keyframe should have its width + // and height set we set it here in the case of it being supplied + // out of band. + video_header.width = sps->second.width; + video_header.height = sps->second.height; + + // If the SPS/PPS was supplied out of band then we will have saved + // the actual bitstream in `data`. + if (sps->second.payload && pps->second.payload) { + RTC_DCHECK_GT(sps->second.size, 0); + RTC_DCHECK_GT(pps->second.size, 0); + prepend_sps_pps = true; + } + } + break; + } + default: + break; + } + } + + RTC_CHECK(!prepend_sps_pps || + (sps != sps_data_.end() && pps != pps_data_.end())); + + // Insert SPS and PPS if they are missing. + if (prepend_sps_pps) { + // Insert SPS. + result.AppendData(kStartCode); + result.AppendData(sps->second.payload.get(), sps->second.size); + + // Insert PPS. + result.AppendData(kStartCode); + result.AppendData(pps->second.payload.get(), pps->second.size); + + // Update codec header to reflect the newly added SPS and PPS. + NaluInfo sps_info; + sps_info.type = H264::NaluType::kSps; + sps_info.sps_id = sps->first; + sps_info.pps_id = -1; + NaluInfo pps_info; + pps_info.type = H264::NaluType::kPps; + pps_info.sps_id = sps->first; + pps_info.pps_id = pps->first; + if (h264_header.nalus_length + 2 <= kMaxNalusPerPacket) { + h264_header.nalus[h264_header.nalus_length++] = sps_info; + h264_header.nalus[h264_header.nalus_length++] = pps_info; + } else { + RTC_LOG(LS_WARNING) + << "Not enough space in H.264 codec header to insert " + "SPS/PPS provided out-of-band."; + } + } + } + + // Insert start code. + switch (h264_header.packetization_type) { + case kH264StapA: { + const uint8_t* payload_end = + packet.video_payload.data() + packet.video_payload.size(); + const uint8_t* nalu_ptr = packet.video_payload.data() + 1; + while (nalu_ptr < payload_end - 1) { + // The first two bytes describe the length of the segment, where a + // segment is the nalu type plus nalu payload. + uint16_t segment_length = nalu_ptr[0] << 8 | nalu_ptr[1]; + nalu_ptr += 2; + + if (nalu_ptr + segment_length <= payload_end) { + result.AppendData(kStartCode); + result.AppendData(nalu_ptr, segment_length); + } + nalu_ptr += segment_length; + } + packet.video_payload = result; + return true; + } + + case kH264FuA: { + if (IsFirstPacketOfFragment(h264_header)) { + result.AppendData(kStartCode); + } + result.AppendData(packet.video_payload); + packet.video_payload = result; + return true; + } + + case kH264SingleNalu: { + result.AppendData(kStartCode); + result.AppendData(packet.video_payload); + packet.video_payload = result; + return true; + } + } + + RTC_DCHECK_NOTREACHED(); + return false; +} + } // namespace webrtc diff --git a/modules/video_coding/h26x_packet_buffer.h b/modules/video_coding/h26x_packet_buffer.h index 21601562c5..8bfae71f7b 100644 --- a/modules/video_coding/h26x_packet_buffer.h +++ b/modules/video_coding/h26x_packet_buffer.h @@ -12,7 +12,9 @@ #define MODULES_VIDEO_CODING_H26X_PACKET_BUFFER_H_ #include +#include #include +#include #include #include "absl/base/attributes.h" @@ -36,20 +38,68 @@ class H26xPacketBuffer { ABSL_MUST_USE_RESULT InsertResult InsertPacket(std::unique_ptr packet); + // Out of band supplied codec parameters for H.264. + void SetSpropParameterSets(const std::string& sprop_parameter_sets); + private: + // Stores PPS payload and the active SPS ID. + struct PpsInfo { + PpsInfo() = default; + PpsInfo(PpsInfo&& rhs) = default; + PpsInfo& operator=(PpsInfo&& rhs) = default; + ~PpsInfo() = default; + + // The value of sps_seq_parameter_set_id for the active SPS. + uint32_t sps_id = 0; + // Payload size. + size_t size = 0; + std::unique_ptr payload; + }; + + // Stores SPS payload and picture size. + struct SpsInfo { + SpsInfo() = default; + SpsInfo(SpsInfo&& rhs) = default; + SpsInfo& operator=(SpsInfo&& rhs) = default; + ~SpsInfo() = default; + + // The width and height of decoded pictures. + int width = -1; + int height = -1; + // Payload size. + size_t size = 0; + std::unique_ptr payload; + }; + static constexpr int kBufferSize = 2048; std::unique_ptr& GetPacket(int64_t unwrapped_seq_num); bool BeginningOfStream(const Packet& packet) const; - std::vector> FindFrames(int64_t unwrapped_seq_num); + InsertResult FindFrames(int64_t unwrapped_seq_num); bool MaybeAssembleFrame(int64_t start_seq_num_unwrapped, int64_t end_sequence_number_unwrapped, - std::vector>& packets); + InsertResult& result); + // Store SPS and PPS nalus. They will be used later when an IDR frame is + // received without SPS/PPS. + void InsertSpsPpsNalus(const std::vector& sps, + const std::vector& pps); + // Insert start code and paramter sets for H.264 payload, also update header + // if parameter sets are inserted. Return false if required SPS or PPS is not + // found. + bool FixH264Packet(Packet& packet); + // Indicates whether IDR frames without SPS and PPS are allowed. const bool h264_idr_only_keyframes_allowed_; std::array, kBufferSize> buffer_; absl::optional last_continuous_unwrapped_seq_num_; SeqNumUnwrapper seq_num_unwrapper_; + + // Map from pps_pic_parameter_set_id to the PPS payload associated with this + // ID. + std::map pps_data_; + // Map from sps_video_parameter_set_id to the SPS payload associated with this + // ID. + std::map sps_data_; }; } // namespace webrtc diff --git a/modules/video_coding/h26x_packet_buffer_unittest.cc b/modules/video_coding/h26x_packet_buffer_unittest.cc index ac5bcb735b..8d6d69136d 100644 --- a/modules/video_coding/h26x_packet_buffer_unittest.cc +++ b/modules/video_coding/h26x_packet_buffer_unittest.cc @@ -42,6 +42,11 @@ using H264::NaluType::kSps; using H264::NaluType::kStapA; constexpr int kBufferSize = 2048; +// Example sprop string from https://tools.ietf.org/html/rfc3984. +const char kExampleSpropString[] = "Z0IACpZTBYmI,aMljiA=="; +static const std::vector kExampleSpropRawSps{ + 0x67, 0x42, 0x00, 0x0A, 0x96, 0x53, 0x05, 0x89, 0x88}; +static const std::vector kExampleSpropRawPps{0x68, 0xC9, 0x63, 0x88}; std::vector StartCode() { return {0, 0, 0, 1}; @@ -59,12 +64,14 @@ class H264Packet { public: explicit H264Packet(H264PacketizationTypes type); - H264Packet& Idr(std::vector payload = {9, 9, 9}); + H264Packet& Idr(std::vector payload = {9, 9, 9}, int pps_id = -1); H264Packet& Slice(std::vector payload = {9, 9, 9}); - H264Packet& Sps(std::vector payload = {9, 9, 9}); + H264Packet& Sps(std::vector payload = {9, 9, 9}, int sps_id = -1); H264Packet& SpsWithResolution(RenderResolution resolution, std::vector payload = {9, 9, 9}); - H264Packet& Pps(std::vector payload = {9, 9, 9}); + H264Packet& Pps(std::vector payload = {9, 9, 9}, + int pps_id = -1, + int sps_id = -1); H264Packet& Aud(); H264Packet& Marker(); H264Packet& AsFirstFragment(); @@ -98,9 +105,11 @@ H264Packet::H264Packet(H264PacketizationTypes type) : type_(type) { video_header_.video_type_header.emplace(); } -H264Packet& H264Packet::Idr(std::vector payload) { +H264Packet& H264Packet::Idr(std::vector payload, int pps_id) { auto& h264_header = H264Header(); - h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kIdr); + auto nalu_info = MakeNaluInfo(kIdr); + nalu_info.pps_id = pps_id; + h264_header.nalus[h264_header.nalus_length++] = nalu_info; nalu_payloads_.push_back(std::move(payload)); return *this; } @@ -112,9 +121,11 @@ H264Packet& H264Packet::Slice(std::vector payload) { return *this; } -H264Packet& H264Packet::Sps(std::vector payload) { +H264Packet& H264Packet::Sps(std::vector payload, int sps_id) { auto& h264_header = H264Header(); - h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kSps); + auto nalu_info = MakeNaluInfo(kSps); + nalu_info.pps_id = sps_id; + h264_header.nalus[h264_header.nalus_length++] = nalu_info; nalu_payloads_.push_back(std::move(payload)); return *this; } @@ -129,9 +140,14 @@ H264Packet& H264Packet::SpsWithResolution(RenderResolution resolution, return *this; } -H264Packet& H264Packet::Pps(std::vector payload) { +H264Packet& H264Packet::Pps(std::vector payload, + int pps_id, + int sps_id) { auto& h264_header = H264Header(); - h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kPps); + auto nalu_info = MakeNaluInfo(kPps); + nalu_info.pps_id = pps_id; + nalu_info.sps_id = sps_id; + h264_header.nalus[h264_header.nalus_length++] = nalu_info; nalu_payloads_.push_back(std::move(payload)); return *this; } @@ -354,14 +370,97 @@ std::vector FlatVector( return res; } -TEST(H26xPacketBufferTest, IdrIsKeyframe) { +TEST(H26xPacketBufferTest, IdrOnlyKeyframeWithSprop) { + H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/true); + packet_buffer.SetSpropParameterSets(kExampleSpropString); + + auto packets = + packet_buffer + .InsertPacket( + H264Packet(kH264SingleNalu).Idr({1, 2, 3}, 0).Marker().Build()) + .packets; + EXPECT_THAT(packets, SizeIs(1)); + EXPECT_THAT(PacketPayload(packets[0]), + ElementsAreArray(FlatVector({StartCode(), + kExampleSpropRawSps, + StartCode(), + kExampleSpropRawPps, + StartCode(), + {kIdr, 1, 2, 3}}))); +} + +TEST(H26xPacketBufferTest, IdrOnlyKeyframeWithoutSprop) { H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/true); - EXPECT_THAT( + // Cannot fix biststream by prepending SPS and PPS because no sprop string is + // available. Request a key frame. + EXPECT_TRUE( packet_buffer - .InsertPacket(H264Packet(kH264SingleNalu).Idr().Marker().Build()) - .packets, - SizeIs(1)); + .InsertPacket( + H264Packet(kH264SingleNalu).Idr({9, 9, 9}, 0).Marker().Build()) + .buffer_cleared); +} + +TEST(H26xPacketBufferTest, IdrOnlyKeyframeWithSpropAndUnknownPpsId) { + H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/true); + packet_buffer.SetSpropParameterSets(kExampleSpropString); + + // Cannot fix biststream because sprop string doesn't contain a PPS with given + // ID. Request a key frame. + EXPECT_TRUE( + packet_buffer + .InsertPacket( + H264Packet(kH264SingleNalu).Idr({9, 9, 9}, 1).Marker().Build()) + .buffer_cleared); +} + +TEST(H26xPacketBufferTest, IdrOnlyKeyframeInTheMiddle) { + H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/true); + packet_buffer.SetSpropParameterSets(kExampleSpropString); + + RTC_UNUSED(packet_buffer.InsertPacket( + H264Packet(kH264SingleNalu).Sps({1, 2, 3}, 1).SeqNum(0).Time(0).Build())); + RTC_UNUSED(packet_buffer.InsertPacket(H264Packet(kH264SingleNalu) + .Pps({4, 5, 6}, 1, 1) + .SeqNum(1) + .Time(0) + .Build())); + EXPECT_THAT(packet_buffer + .InsertPacket(H264Packet(kH264SingleNalu) + .Idr({7, 8, 9}, 1) + .SeqNum(2) + .Time(0) + .Marker() + .Build()) + .packets, + SizeIs(3)); + + EXPECT_THAT(packet_buffer + .InsertPacket(H264Packet(kH264SingleNalu) + .Slice() + .SeqNum(3) + .Time(1) + .Marker() + .Build()) + .packets, + SizeIs(1)); + + auto packets = packet_buffer + .InsertPacket(H264Packet(kH264SingleNalu) + .Idr({10, 11, 12}, 0) + .SeqNum(4) + .Time(2) + .Marker() + .Build()) + .packets; + EXPECT_THAT(packets, SizeIs(1)); + EXPECT_THAT(PacketPayload(packets[0]), + ElementsAreArray(FlatVector({StartCode(), + kExampleSpropRawSps, + StartCode(), + kExampleSpropRawPps, + StartCode(), + {kIdr, 10, 11, 12}}))); } TEST(H26xPacketBufferTest, IdrIsNotKeyframe) { @@ -376,6 +475,7 @@ TEST(H26xPacketBufferTest, IdrIsNotKeyframe) { TEST(H26xPacketBufferTest, IdrIsKeyframeFuaRequiresFirstFragmet) { H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/true); + packet_buffer.SetSpropParameterSets(kExampleSpropString); // Not marked as the first fragment EXPECT_THAT( @@ -394,7 +494,7 @@ TEST(H26xPacketBufferTest, IdrIsKeyframeFuaRequiresFirstFragmet) { // Marked as first fragment EXPECT_THAT(packet_buffer .InsertPacket(H264Packet(kH264FuA) - .Idr() + .Idr({9, 9, 9}, 0) .SeqNum(2) .Time(1) .AsFirstFragment() @@ -428,6 +528,37 @@ TEST(H26xPacketBufferTest, SpsPpsIdrIsKeyframeSingleNalus) { SizeIs(3)); } +TEST(H26xPacketBufferTest, SpsPpsIdrIsKeyframeIgnoresSprop) { + H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/false); + + // When h264_allow_idr_only_keyframes is false, sprop string should be + // ignored. Use in band parameter sets. + packet_buffer.SetSpropParameterSets(kExampleSpropString); + + RTC_UNUSED(packet_buffer.InsertPacket( + H264Packet(kH264SingleNalu).Sps({1, 2, 3}, 0).SeqNum(0).Time(0).Build())); + RTC_UNUSED(packet_buffer.InsertPacket(H264Packet(kH264SingleNalu) + .Pps({4, 5, 6}, 0, 0) + .SeqNum(1) + .Time(0) + .Build())); + auto packets = packet_buffer + .InsertPacket(H264Packet(kH264SingleNalu) + .Idr({7, 8, 9}, 0) + .SeqNum(2) + .Time(0) + .Marker() + .Build()) + .packets; + EXPECT_THAT(packets, SizeIs(3)); + EXPECT_THAT(PacketPayload(packets[0]), + ElementsAreArray(FlatVector({StartCode(), {kSps, 1, 2, 3}}))); + EXPECT_THAT(PacketPayload(packets[1]), + ElementsAreArray(FlatVector({StartCode(), {kPps, 4, 5, 6}}))); + EXPECT_THAT(PacketPayload(packets[2]), + ElementsAreArray(FlatVector({StartCode(), {kIdr, 7, 8, 9}}))); +} + TEST(H26xPacketBufferTest, PpsIdrIsNotKeyframeSingleNalus) { H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/false); @@ -966,6 +1097,15 @@ TEST(H26xPacketBufferTest, H265IdrIsNotKeyFrame) { IsEmpty()); } +TEST(H26xPacketBufferTest, H265IdrIsNotKeyFrameEvenWithSprop) { + H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/true); + packet_buffer.SetSpropParameterSets(kExampleSpropString); + + EXPECT_THAT( + packet_buffer.InsertPacket(H265Packet().Idr().Marker().Build()).packets, + IsEmpty()); +} + TEST(H26xPacketBufferTest, H265SpsPpsIdrIsNotKeyFrame) { H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/false); diff --git a/video/BUILD.gn b/video/BUILD.gn index 8752929fa2..9f20746c1f 100644 --- a/video/BUILD.gn +++ b/video/BUILD.gn @@ -128,6 +128,7 @@ rtc_library("video") { "../modules/rtp_rtcp:rtp_rtcp_format", "../modules/rtp_rtcp:rtp_video_header", "../modules/video_coding", + "../modules/video_coding:h264_sprop_parameter_sets", "../modules/video_coding:nack_requester", "../modules/video_coding:packet_buffer", "../modules/video_coding:video_codec_interface",