H26xPacketBuffer handles out of band H.264 parameter sets.

This CL updates H26xPacketBuffer to store and prepend SPS and PPS for
H.264 bitstreams when IDR only keyframe is allowed.

Bug: webrtc:13485
Change-Id: Ic1edc623dff568d54d3ce29b42dd8eab3312f5cb
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/342225
Reviewed-by: Philip Eliasson <philipel@webrtc.org>
Commit-Queue: Philip Eliasson <philipel@webrtc.org>
Reviewed-by: Sergey Silkin <ssilkin@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#41986}

modules/video_coding/BUILD.gn

@@ -130,6 +130,7 @@ rtc_library("h26x_packet_buffer") {
   ]
   deps = [
     ":codec_globals_headers",
+    ":h264_sprop_parameter_sets",
     ":packet_buffer",
     "../../api:array_view",
     "../../api:rtp_packet_info",
@@ -162,6 +163,17 @@ rtc_library("frame_helpers") {
   absl_deps = [ "//third_party/abseil-cpp/absl/container:inlined_vector" ]
 }
 
+rtc_library("h264_sprop_parameter_sets") {
+  sources = [
+    "h264_sprop_parameter_sets.cc",
+    "h264_sprop_parameter_sets.h",
+  ]
+  deps = [
+    "../../rtc_base:logging",
+    "../../rtc_base/third_party/base64",
+  ]
+}
+
 rtc_library("video_coding") {
   visibility = [ "*" ]
   sources = [
@@ -172,8 +184,6 @@ rtc_library("video_coding") {
     "fec_rate_table.h",
     "generic_decoder.cc",
     "generic_decoder.h",
-    "h264_sprop_parameter_sets.cc",
-    "h264_sprop_parameter_sets.h",
     "h264_sps_pps_tracker.cc",
     "h264_sps_pps_tracker.h",
     "include/video_codec_initializer.h",
@@ -203,6 +213,7 @@ rtc_library("video_coding") {
     ":codec_globals_headers",
     ":encoded_frame",
     ":frame_helpers",
+    ":h264_sprop_parameter_sets",
     ":video_codec_interface",
     ":video_coding_utility",
     ":webrtc_vp8_scalability",
@@ -1173,6 +1184,7 @@ if (rtc_include_tests) {
       ":encoded_frame",
       ":frame_dependencies_calculator",
       ":frame_helpers",
+      ":h264_sprop_parameter_sets",
       ":h26x_packet_buffer",
       ":nack_requester",
       ":packet_buffer",

modules/video_coding/h26x_packet_buffer.cc

@@ -19,10 +19,13 @@
 #include "api/rtp_packet_info.h"
 #include "api/video/video_frame_type.h"
 #include "common_video/h264/h264_common.h"
+#include "common_video/h264/pps_parser.h"
+#include "common_video/h264/sps_parser.h"
 #include "modules/rtp_rtcp/source/rtp_header_extensions.h"
 #include "modules/rtp_rtcp/source/rtp_packet_received.h"
 #include "modules/rtp_rtcp/source/rtp_video_header.h"
 #include "modules/video_coding/codecs/h264/include/h264_globals.h"
+#include "modules/video_coding/h264_sprop_parameter_sets.h"
 #include "rtc_base/checks.h"
 #include "rtc_base/copy_on_write_buffer.h"
 #include "rtc_base/logging.h"
@@ -91,55 +94,6 @@ bool HasVps(const H26xPacketBuffer::Packet& packet) {
 }
 #endif
 
-// TODO(bugs.webrtc.org/13157): Update the H264 depacketizer so we don't have to
-//                              fiddle with the payload at this point.
-rtc::CopyOnWriteBuffer FixH264VideoPayload(
-    rtc::ArrayView<const uint8_t> payload,
-    const RTPVideoHeader& video_header) {
-  constexpr uint8_t kStartCode[] = {0, 0, 0, 1};
-
-  const auto& h264_header =
-      absl::get<RTPVideoHeaderH264>(video_header.video_type_header);
-
-  rtc::CopyOnWriteBuffer result;
-  switch (h264_header.packetization_type) {
-    case kH264StapA: {
-      const uint8_t* payload_end = payload.data() + payload.size();
-      const uint8_t* nalu_ptr = payload.data() + 1;
-      while (nalu_ptr < payload_end - 1) {
-        // The first two bytes describe the length of the segment, where a
-        // segment is the nalu type plus nalu payload.
-        uint16_t segment_length = nalu_ptr[0] << 8 | nalu_ptr[1];
-        nalu_ptr += 2;
-
-        if (nalu_ptr + segment_length <= payload_end) {
-          result.AppendData(kStartCode);
-          result.AppendData(nalu_ptr, segment_length);
-        }
-        nalu_ptr += segment_length;
-      }
-      return result;
-    }
-
-    case kH264FuA: {
-      if (IsFirstPacketOfFragment(h264_header)) {
-        result.AppendData(kStartCode);
-      }
-      result.AppendData(payload);
-      return result;
-    }
-
-    case kH264SingleNalu: {
-      result.AppendData(kStartCode);
-      result.AppendData(payload);
-      return result;
-    }
-  }
-
-  RTC_DCHECK_NOTREACHED();
-  return result;
-}
-
 }  // namespace
 
 H26xPacketBuffer::H26xPacketBuffer(bool h264_idr_only_keyframes_allowed)
@@ -162,8 +116,7 @@ H26xPacketBuffer::InsertResult H26xPacketBuffer::InsertPacket(
     packet_slot = std::move(packet);
   }
 
-  result.packets = FindFrames(unwrapped_seq_num);
+  return FindFrames(unwrapped_seq_num);
-  return result;
 }
 
 std::unique_ptr<H26xPacketBuffer::Packet>& H26xPacketBuffer::GetPacket(
@@ -185,9 +138,9 @@ bool H26xPacketBuffer::BeginningOfStream(
   return false;
 }
 
-std::vector<std::unique_ptr<H26xPacketBuffer::Packet>>
+H26xPacketBuffer::InsertResult H26xPacketBuffer::FindFrames(
-H26xPacketBuffer::FindFrames(int64_t unwrapped_seq_num) {
+    int64_t unwrapped_seq_num) {
-  std::vector<std::unique_ptr<Packet>> found_frames;
+  InsertResult result;
 
   Packet* packet = GetPacket(unwrapped_seq_num).get();
   RTC_CHECK(packet != nullptr);
@@ -197,7 +150,7 @@ H26xPacketBuffer::FindFrames(int64_t unwrapped_seq_num) {
   if (unwrapped_seq_num - 1 != last_continuous_unwrapped_seq_num_) {
     if (unwrapped_seq_num <= last_continuous_unwrapped_seq_num_ ||
         !BeginningOfStream(*packet)) {
-      return found_frames;
+      return result;
     }
 
     last_continuous_unwrapped_seq_num_ = unwrapped_seq_num;
@@ -211,7 +164,7 @@ H26xPacketBuffer::FindFrames(int64_t unwrapped_seq_num) {
     // the 'buffer_'. Check that the `packet` sequence number match the expected
     // unwrapped sequence number.
     if (static_cast<uint16_t>(seq_num) != packet->seq_num) {
-      return found_frames;
+      return result;
     }
 
     last_continuous_unwrapped_seq_num_ = seq_num;
@@ -225,12 +178,12 @@ H26xPacketBuffer::FindFrames(int64_t unwrapped_seq_num) {
         auto& prev_packet = GetPacket(seq_num_start - 1);
 
         if (prev_packet == nullptr || prev_packet->timestamp != rtp_timestamp) {
-          if (MaybeAssembleFrame(seq_num_start, seq_num, found_frames)) {
+          if (MaybeAssembleFrame(seq_num_start, seq_num, result)) {
             // Frame was assembled, continue to look for more frames.
             break;
           } else {
             // Frame was not assembled, no subsequent frame will be continuous.
-            return found_frames;
+            return result;
           }
         }
       }
@@ -239,24 +192,23 @@ H26xPacketBuffer::FindFrames(int64_t unwrapped_seq_num) {
     seq_num++;
     packet = GetPacket(seq_num).get();
     if (packet == nullptr) {
-      return found_frames;
+      return result;
     }
   }
 
-  return found_frames;
+  return result;
 }
 
-bool H26xPacketBuffer::MaybeAssembleFrame(
+bool H26xPacketBuffer::MaybeAssembleFrame(int64_t start_seq_num_unwrapped,
-    int64_t start_seq_num_unwrapped,
                                           int64_t end_sequence_number_unwrapped,
-    std::vector<std::unique_ptr<Packet>>& frames) {
+                                          InsertResult& result) {
 #ifdef RTC_ENABLE_H265
   bool has_vps = false;
 #endif
   bool has_sps = false;
   bool has_pps = false;
+  // Includes IDR, CRA and BLA for HEVC.
   bool has_idr = false;
-  bool has_irap = false;
 
   int width = -1;
   int height = -1;
@@ -284,13 +236,13 @@ bool H26xPacketBuffer::MaybeAssembleFrame(
       for (const auto& nalu_index : nalu_indices) {
         uint8_t nalu_type = H265::ParseNaluType(
             packet->video_payload.cdata()[nalu_index.payload_start_offset]);
-        has_irap |= (nalu_type >= H265::NaluType::kBlaWLp &&
+        has_idr |= (nalu_type >= H265::NaluType::kBlaWLp &&
                     nalu_type <= H265::NaluType::kRsvIrapVcl23);
         has_vps |= nalu_type == H265::NaluType::kVps;
         has_sps |= nalu_type == H265::NaluType::kSps;
         has_pps |= nalu_type == H265::NaluType::kPps;
       }
-      if (has_irap) {
+      if (has_idr) {
         if (!has_vps || !has_sps || !has_pps) {
           return false;
         }
@@ -317,21 +269,253 @@ bool H26xPacketBuffer::MaybeAssembleFrame(
         packet->video_header.height = height;
       }
 
-      packet->video_header.frame_type = has_idr || has_irap
+      packet->video_header.frame_type = has_idr
                                             ? VideoFrameType::kVideoFrameKey
                                             : VideoFrameType::kVideoFrameDelta;
     }
 
-    // Start code is inserted by depacktizer for H.265.
+    // Only applies to H.264 because start code is inserted by depacktizer for
+    // H.265 and out-of-band parameter sets is not supported by H.265.
     if (packet->codec() == kVideoCodecH264) {
-      packet->video_payload =
+      if (!FixH264Packet(*packet)) {
-          FixH264VideoPayload(packet->video_payload, packet->video_header);
+        // The buffer is not cleared actually, but a key frame request is
+        // needed.
+        result.buffer_cleared = true;
+        return false;
+      }
     }
 
-    frames.push_back(std::move(packet));
+    result.packets.push_back(std::move(packet));
   }
 
   return true;
 }
 
+void H26xPacketBuffer::SetSpropParameterSets(
+    const std::string& sprop_parameter_sets) {
+  if (!h264_idr_only_keyframes_allowed_) {
+    RTC_LOG(LS_WARNING) << "Ignore sprop parameter sets because IDR only "
+                           "keyframe is not allowed.";
+    return;
+  }
+  H264SpropParameterSets sprop_decoder;
+  if (!sprop_decoder.DecodeSprop(sprop_parameter_sets)) {
+    return;
+  }
+  InsertSpsPpsNalus(sprop_decoder.sps_nalu(), sprop_decoder.pps_nalu());
+}
+
+void H26xPacketBuffer::InsertSpsPpsNalus(const std::vector<uint8_t>& sps,
+                                         const std::vector<uint8_t>& pps) {
+  RTC_CHECK(h264_idr_only_keyframes_allowed_);
+  constexpr size_t kNaluHeaderOffset = 1;
+  if (sps.size() < kNaluHeaderOffset) {
+    RTC_LOG(LS_WARNING) << "SPS size  " << sps.size() << " is smaller than "
+                        << kNaluHeaderOffset;
+    return;
+  }
+  if ((sps[0] & 0x1f) != H264::NaluType::kSps) {
+    RTC_LOG(LS_WARNING) << "SPS Nalu header missing";
+    return;
+  }
+  if (pps.size() < kNaluHeaderOffset) {
+    RTC_LOG(LS_WARNING) << "PPS size  " << pps.size() << " is smaller than "
+                        << kNaluHeaderOffset;
+    return;
+  }
+  if ((pps[0] & 0x1f) != H264::NaluType::kPps) {
+    RTC_LOG(LS_WARNING) << "SPS Nalu header missing";
+    return;
+  }
+  absl::optional<SpsParser::SpsState> parsed_sps = SpsParser::ParseSps(
+      sps.data() + kNaluHeaderOffset, sps.size() - kNaluHeaderOffset);
+  absl::optional<PpsParser::PpsState> parsed_pps = PpsParser::ParsePps(
+      pps.data() + kNaluHeaderOffset, pps.size() - kNaluHeaderOffset);
+
+  if (!parsed_sps) {
+    RTC_LOG(LS_WARNING) << "Failed to parse SPS.";
+  }
+
+  if (!parsed_pps) {
+    RTC_LOG(LS_WARNING) << "Failed to parse PPS.";
+  }
+
+  if (!parsed_pps || !parsed_sps) {
+    return;
+  }
+
+  SpsInfo sps_info;
+  sps_info.size = sps.size();
+  sps_info.width = parsed_sps->width;
+  sps_info.height = parsed_sps->height;
+  uint8_t* sps_data = new uint8_t[sps_info.size];
+  memcpy(sps_data, sps.data(), sps_info.size);
+  sps_info.payload.reset(sps_data);
+  sps_data_[parsed_sps->id] = std::move(sps_info);
+
+  PpsInfo pps_info;
+  pps_info.size = pps.size();
+  pps_info.sps_id = parsed_pps->sps_id;
+  uint8_t* pps_data = new uint8_t[pps_info.size];
+  memcpy(pps_data, pps.data(), pps_info.size);
+  pps_info.payload.reset(pps_data);
+  pps_data_[parsed_pps->id] = std::move(pps_info);
+
+  RTC_LOG(LS_INFO) << "Inserted SPS id " << parsed_sps->id << " and PPS id "
+                   << parsed_pps->id << " (referencing SPS "
+                   << parsed_pps->sps_id << ")";
+}
+
+// TODO(bugs.webrtc.org/13157): Update the H264 depacketizer so we don't have to
+//                              fiddle with the payload at this point.
+bool H26xPacketBuffer::FixH264Packet(Packet& packet) {
+  constexpr uint8_t kStartCode[] = {0, 0, 0, 1};
+
+  RTPVideoHeader& video_header = packet.video_header;
+  RTPVideoHeaderH264& h264_header =
+      absl::get<RTPVideoHeaderH264>(video_header.video_type_header);
+
+  rtc::CopyOnWriteBuffer result;
+
+  if (h264_idr_only_keyframes_allowed_) {
+    // Check if sps and pps insertion is needed.
+    bool prepend_sps_pps = false;
+    auto sps = sps_data_.end();
+    auto pps = pps_data_.end();
+
+    for (size_t i = 0; i < h264_header.nalus_length; ++i) {
+      const NaluInfo& nalu = h264_header.nalus[i];
+      switch (nalu.type) {
+        case H264::NaluType::kSps: {
+          SpsInfo& sps_info = sps_data_[nalu.sps_id];
+          sps_info.width = video_header.width;
+          sps_info.height = video_header.height;
+          break;
+        }
+        case H264::NaluType::kPps: {
+          pps_data_[nalu.pps_id].sps_id = nalu.sps_id;
+          break;
+        }
+        case H264::NaluType::kIdr: {
+          // If this is the first packet of an IDR, make sure we have the
+          // required SPS/PPS and also calculate how much extra space we need
+          // in the buffer to prepend the SPS/PPS to the bitstream with start
+          // codes.
+          if (video_header.is_first_packet_in_frame) {
+            if (nalu.pps_id == -1) {
+              RTC_LOG(LS_WARNING) << "No PPS id in IDR nalu.";
+              return false;
+            }
+
+            pps = pps_data_.find(nalu.pps_id);
+            if (pps == pps_data_.end()) {
+              RTC_LOG(LS_WARNING)
+                  << "No PPS with id << " << nalu.pps_id << " received";
+              return false;
+            }
+
+            sps = sps_data_.find(pps->second.sps_id);
+            if (sps == sps_data_.end()) {
+              RTC_LOG(LS_WARNING)
+                  << "No SPS with id << " << pps->second.sps_id << " received";
+              return false;
+            }
+
+            // Since the first packet of every keyframe should have its width
+            // and height set we set it here in the case of it being supplied
+            // out of band.
+            video_header.width = sps->second.width;
+            video_header.height = sps->second.height;
+
+            // If the SPS/PPS was supplied out of band then we will have saved
+            // the actual bitstream in `data`.
+            if (sps->second.payload && pps->second.payload) {
+              RTC_DCHECK_GT(sps->second.size, 0);
+              RTC_DCHECK_GT(pps->second.size, 0);
+              prepend_sps_pps = true;
+            }
+          }
+          break;
+        }
+        default:
+          break;
+      }
+    }
+
+    RTC_CHECK(!prepend_sps_pps ||
+              (sps != sps_data_.end() && pps != pps_data_.end()));
+
+    // Insert SPS and PPS if they are missing.
+    if (prepend_sps_pps) {
+      // Insert SPS.
+      result.AppendData(kStartCode);
+      result.AppendData(sps->second.payload.get(), sps->second.size);
+
+      // Insert PPS.
+      result.AppendData(kStartCode);
+      result.AppendData(pps->second.payload.get(), pps->second.size);
+
+      // Update codec header to reflect the newly added SPS and PPS.
+      NaluInfo sps_info;
+      sps_info.type = H264::NaluType::kSps;
+      sps_info.sps_id = sps->first;
+      sps_info.pps_id = -1;
+      NaluInfo pps_info;
+      pps_info.type = H264::NaluType::kPps;
+      pps_info.sps_id = sps->first;
+      pps_info.pps_id = pps->first;
+      if (h264_header.nalus_length + 2 <= kMaxNalusPerPacket) {
+        h264_header.nalus[h264_header.nalus_length++] = sps_info;
+        h264_header.nalus[h264_header.nalus_length++] = pps_info;
+      } else {
+        RTC_LOG(LS_WARNING)
+            << "Not enough space in H.264 codec header to insert "
+               "SPS/PPS provided out-of-band.";
+      }
+    }
+  }
+
+  // Insert start code.
+  switch (h264_header.packetization_type) {
+    case kH264StapA: {
+      const uint8_t* payload_end =
+          packet.video_payload.data() + packet.video_payload.size();
+      const uint8_t* nalu_ptr = packet.video_payload.data() + 1;
+      while (nalu_ptr < payload_end - 1) {
+        // The first two bytes describe the length of the segment, where a
+        // segment is the nalu type plus nalu payload.
+        uint16_t segment_length = nalu_ptr[0] << 8 | nalu_ptr[1];
+        nalu_ptr += 2;
+
+        if (nalu_ptr + segment_length <= payload_end) {
+          result.AppendData(kStartCode);
+          result.AppendData(nalu_ptr, segment_length);
+        }
+        nalu_ptr += segment_length;
+      }
+      packet.video_payload = result;
+      return true;
+    }
+
+    case kH264FuA: {
+      if (IsFirstPacketOfFragment(h264_header)) {
+        result.AppendData(kStartCode);
+      }
+      result.AppendData(packet.video_payload);
+      packet.video_payload = result;
+      return true;
+    }
+
+    case kH264SingleNalu: {
+      result.AppendData(kStartCode);
+      result.AppendData(packet.video_payload);
+      packet.video_payload = result;
+      return true;
+    }
+  }
+
+  RTC_DCHECK_NOTREACHED();
+  return false;
+}
+
 }  // namespace webrtc

modules/video_coding/h26x_packet_buffer.h

@@ -12,7 +12,9 @@
 #define MODULES_VIDEO_CODING_H26X_PACKET_BUFFER_H_
 
 #include <array>
+#include <map>
 #include <memory>
+#include <string>
 #include <vector>
 
 #include "absl/base/attributes.h"
@@ -36,20 +38,68 @@ class H26xPacketBuffer {
   ABSL_MUST_USE_RESULT InsertResult
   InsertPacket(std::unique_ptr<Packet> packet);
 
+  // Out of band supplied codec parameters for H.264.
+  void SetSpropParameterSets(const std::string& sprop_parameter_sets);
+
  private:
+  // Stores PPS payload and the active SPS ID.
+  struct PpsInfo {
+    PpsInfo() = default;
+    PpsInfo(PpsInfo&& rhs) = default;
+    PpsInfo& operator=(PpsInfo&& rhs) = default;
+    ~PpsInfo() = default;
+
+    // The value of sps_seq_parameter_set_id for the active SPS.
+    uint32_t sps_id = 0;
+    // Payload size.
+    size_t size = 0;
+    std::unique_ptr<uint8_t[]> payload;
+  };
+
+  // Stores SPS payload and picture size.
+  struct SpsInfo {
+    SpsInfo() = default;
+    SpsInfo(SpsInfo&& rhs) = default;
+    SpsInfo& operator=(SpsInfo&& rhs) = default;
+    ~SpsInfo() = default;
+
+    // The width and height of decoded pictures.
+    int width = -1;
+    int height = -1;
+    // Payload size.
+    size_t size = 0;
+    std::unique_ptr<uint8_t[]> payload;
+  };
+
   static constexpr int kBufferSize = 2048;
 
   std::unique_ptr<Packet>& GetPacket(int64_t unwrapped_seq_num);
   bool BeginningOfStream(const Packet& packet) const;
-  std::vector<std::unique_ptr<Packet>> FindFrames(int64_t unwrapped_seq_num);
+  InsertResult FindFrames(int64_t unwrapped_seq_num);
   bool MaybeAssembleFrame(int64_t start_seq_num_unwrapped,
                           int64_t end_sequence_number_unwrapped,
-                          std::vector<std::unique_ptr<Packet>>& packets);
+                          InsertResult& result);
+  // Store SPS and PPS nalus. They will be used later when an IDR frame is
+  // received without SPS/PPS.
+  void InsertSpsPpsNalus(const std::vector<uint8_t>& sps,
+                         const std::vector<uint8_t>& pps);
+  // Insert start code and paramter sets for H.264 payload, also update header
+  // if parameter sets are inserted. Return false if required SPS or PPS is not
+  // found.
+  bool FixH264Packet(Packet& packet);
 
+  // Indicates whether IDR frames without SPS and PPS are allowed.
   const bool h264_idr_only_keyframes_allowed_;
   std::array<std::unique_ptr<Packet>, kBufferSize> buffer_;
   absl::optional<int64_t> last_continuous_unwrapped_seq_num_;
   SeqNumUnwrapper<uint16_t> seq_num_unwrapper_;
+
+  // Map from pps_pic_parameter_set_id to the PPS payload associated with this
+  // ID.
+  std::map<uint32_t, PpsInfo> pps_data_;
+  // Map from sps_video_parameter_set_id to the SPS payload associated with this
+  // ID.
+  std::map<uint32_t, SpsInfo> sps_data_;
 };
 
 }  // namespace webrtc

modules/video_coding/h26x_packet_buffer_unittest.cc

@@ -42,6 +42,11 @@ using H264::NaluType::kSps;
 using H264::NaluType::kStapA;
 
 constexpr int kBufferSize = 2048;
+// Example sprop string from https://tools.ietf.org/html/rfc3984.
+const char kExampleSpropString[] = "Z0IACpZTBYmI,aMljiA==";
+static const std::vector<uint8_t> kExampleSpropRawSps{
+    0x67, 0x42, 0x00, 0x0A, 0x96, 0x53, 0x05, 0x89, 0x88};
+static const std::vector<uint8_t> kExampleSpropRawPps{0x68, 0xC9, 0x63, 0x88};
 
 std::vector<uint8_t> StartCode() {
   return {0, 0, 0, 1};
@@ -59,12 +64,14 @@ class H264Packet {
  public:
   explicit H264Packet(H264PacketizationTypes type);
 
-  H264Packet& Idr(std::vector<uint8_t> payload = {9, 9, 9});
+  H264Packet& Idr(std::vector<uint8_t> payload = {9, 9, 9}, int pps_id = -1);
   H264Packet& Slice(std::vector<uint8_t> payload = {9, 9, 9});
-  H264Packet& Sps(std::vector<uint8_t> payload = {9, 9, 9});
+  H264Packet& Sps(std::vector<uint8_t> payload = {9, 9, 9}, int sps_id = -1);
   H264Packet& SpsWithResolution(RenderResolution resolution,
                                 std::vector<uint8_t> payload = {9, 9, 9});
-  H264Packet& Pps(std::vector<uint8_t> payload = {9, 9, 9});
+  H264Packet& Pps(std::vector<uint8_t> payload = {9, 9, 9},
+                  int pps_id = -1,
+                  int sps_id = -1);
   H264Packet& Aud();
   H264Packet& Marker();
   H264Packet& AsFirstFragment();
@@ -98,9 +105,11 @@ H264Packet::H264Packet(H264PacketizationTypes type) : type_(type) {
   video_header_.video_type_header.emplace<RTPVideoHeaderH264>();
 }
 
-H264Packet& H264Packet::Idr(std::vector<uint8_t> payload) {
+H264Packet& H264Packet::Idr(std::vector<uint8_t> payload, int pps_id) {
   auto& h264_header = H264Header();
-  h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kIdr);
+  auto nalu_info = MakeNaluInfo(kIdr);
+  nalu_info.pps_id = pps_id;
+  h264_header.nalus[h264_header.nalus_length++] = nalu_info;
   nalu_payloads_.push_back(std::move(payload));
   return *this;
 }
@@ -112,9 +121,11 @@ H264Packet& H264Packet::Slice(std::vector<uint8_t> payload) {
   return *this;
 }
 
-H264Packet& H264Packet::Sps(std::vector<uint8_t> payload) {
+H264Packet& H264Packet::Sps(std::vector<uint8_t> payload, int sps_id) {
   auto& h264_header = H264Header();
-  h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kSps);
+  auto nalu_info = MakeNaluInfo(kSps);
+  nalu_info.pps_id = sps_id;
+  h264_header.nalus[h264_header.nalus_length++] = nalu_info;
   nalu_payloads_.push_back(std::move(payload));
   return *this;
 }
@@ -129,9 +140,14 @@ H264Packet& H264Packet::SpsWithResolution(RenderResolution resolution,
   return *this;
 }
 
-H264Packet& H264Packet::Pps(std::vector<uint8_t> payload) {
+H264Packet& H264Packet::Pps(std::vector<uint8_t> payload,
+                            int pps_id,
+                            int sps_id) {
   auto& h264_header = H264Header();
-  h264_header.nalus[h264_header.nalus_length++] = MakeNaluInfo(kPps);
+  auto nalu_info = MakeNaluInfo(kPps);
+  nalu_info.pps_id = pps_id;
+  nalu_info.sps_id = sps_id;
+  h264_header.nalus[h264_header.nalus_length++] = nalu_info;
   nalu_payloads_.push_back(std::move(payload));
   return *this;
 }
@@ -354,14 +370,97 @@ std::vector<uint8_t> FlatVector(
   return res;
 }
 
-TEST(H26xPacketBufferTest, IdrIsKeyframe) {
+TEST(H26xPacketBufferTest, IdrOnlyKeyframeWithSprop) {
+  H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/true);
+  packet_buffer.SetSpropParameterSets(kExampleSpropString);
+
+  auto packets =
+      packet_buffer
+          .InsertPacket(
+              H264Packet(kH264SingleNalu).Idr({1, 2, 3}, 0).Marker().Build())
+          .packets;
+  EXPECT_THAT(packets, SizeIs(1));
+  EXPECT_THAT(PacketPayload(packets[0]),
+              ElementsAreArray(FlatVector({StartCode(),
+                                           kExampleSpropRawSps,
+                                           StartCode(),
+                                           kExampleSpropRawPps,
+                                           StartCode(),
+                                           {kIdr, 1, 2, 3}})));
+}
+
+TEST(H26xPacketBufferTest, IdrOnlyKeyframeWithoutSprop) {
   H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/true);
 
-  EXPECT_THAT(
+  // Cannot fix biststream by prepending SPS and PPS because no sprop string is
+  // available. Request a key frame.
+  EXPECT_TRUE(
       packet_buffer
-          .InsertPacket(H264Packet(kH264SingleNalu).Idr().Marker().Build())
+          .InsertPacket(
+              H264Packet(kH264SingleNalu).Idr({9, 9, 9}, 0).Marker().Build())
+          .buffer_cleared);
+}
+
+TEST(H26xPacketBufferTest, IdrOnlyKeyframeWithSpropAndUnknownPpsId) {
+  H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/true);
+  packet_buffer.SetSpropParameterSets(kExampleSpropString);
+
+  // Cannot fix biststream because sprop string doesn't contain a PPS with given
+  // ID. Request a key frame.
+  EXPECT_TRUE(
+      packet_buffer
+          .InsertPacket(
+              H264Packet(kH264SingleNalu).Idr({9, 9, 9}, 1).Marker().Build())
+          .buffer_cleared);
+}
+
+TEST(H26xPacketBufferTest, IdrOnlyKeyframeInTheMiddle) {
+  H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/true);
+  packet_buffer.SetSpropParameterSets(kExampleSpropString);
+
+  RTC_UNUSED(packet_buffer.InsertPacket(
+      H264Packet(kH264SingleNalu).Sps({1, 2, 3}, 1).SeqNum(0).Time(0).Build()));
+  RTC_UNUSED(packet_buffer.InsertPacket(H264Packet(kH264SingleNalu)
+                                            .Pps({4, 5, 6}, 1, 1)
+                                            .SeqNum(1)
+                                            .Time(0)
+                                            .Build()));
+  EXPECT_THAT(packet_buffer
+                  .InsertPacket(H264Packet(kH264SingleNalu)
+                                    .Idr({7, 8, 9}, 1)
+                                    .SeqNum(2)
+                                    .Time(0)
+                                    .Marker()
+                                    .Build())
+                  .packets,
+              SizeIs(3));
+
+  EXPECT_THAT(packet_buffer
+                  .InsertPacket(H264Packet(kH264SingleNalu)
+                                    .Slice()
+                                    .SeqNum(3)
+                                    .Time(1)
+                                    .Marker()
+                                    .Build())
                   .packets,
               SizeIs(1));
+
+  auto packets = packet_buffer
+                     .InsertPacket(H264Packet(kH264SingleNalu)
+                                       .Idr({10, 11, 12}, 0)
+                                       .SeqNum(4)
+                                       .Time(2)
+                                       .Marker()
+                                       .Build())
+                     .packets;
+  EXPECT_THAT(packets, SizeIs(1));
+  EXPECT_THAT(PacketPayload(packets[0]),
+              ElementsAreArray(FlatVector({StartCode(),
+                                           kExampleSpropRawSps,
+                                           StartCode(),
+                                           kExampleSpropRawPps,
+                                           StartCode(),
+                                           {kIdr, 10, 11, 12}})));
 }
 
 TEST(H26xPacketBufferTest, IdrIsNotKeyframe) {
@@ -376,6 +475,7 @@ TEST(H26xPacketBufferTest, IdrIsNotKeyframe) {
 
 TEST(H26xPacketBufferTest, IdrIsKeyframeFuaRequiresFirstFragmet) {
   H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/true);
+  packet_buffer.SetSpropParameterSets(kExampleSpropString);
 
   // Not marked as the first fragment
   EXPECT_THAT(
@@ -394,7 +494,7 @@ TEST(H26xPacketBufferTest, IdrIsKeyframeFuaRequiresFirstFragmet) {
   // Marked as first fragment
   EXPECT_THAT(packet_buffer
                   .InsertPacket(H264Packet(kH264FuA)
-                                    .Idr()
+                                    .Idr({9, 9, 9}, 0)
                                     .SeqNum(2)
                                     .Time(1)
                                     .AsFirstFragment()
@@ -428,6 +528,37 @@ TEST(H26xPacketBufferTest, SpsPpsIdrIsKeyframeSingleNalus) {
               SizeIs(3));
 }
 
+TEST(H26xPacketBufferTest, SpsPpsIdrIsKeyframeIgnoresSprop) {
+  H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/false);
+
+  // When h264_allow_idr_only_keyframes is false, sprop string should be
+  // ignored. Use in band parameter sets.
+  packet_buffer.SetSpropParameterSets(kExampleSpropString);
+
+  RTC_UNUSED(packet_buffer.InsertPacket(
+      H264Packet(kH264SingleNalu).Sps({1, 2, 3}, 0).SeqNum(0).Time(0).Build()));
+  RTC_UNUSED(packet_buffer.InsertPacket(H264Packet(kH264SingleNalu)
+                                            .Pps({4, 5, 6}, 0, 0)
+                                            .SeqNum(1)
+                                            .Time(0)
+                                            .Build()));
+  auto packets = packet_buffer
+                     .InsertPacket(H264Packet(kH264SingleNalu)
+                                       .Idr({7, 8, 9}, 0)
+                                       .SeqNum(2)
+                                       .Time(0)
+                                       .Marker()
+                                       .Build())
+                     .packets;
+  EXPECT_THAT(packets, SizeIs(3));
+  EXPECT_THAT(PacketPayload(packets[0]),
+              ElementsAreArray(FlatVector({StartCode(), {kSps, 1, 2, 3}})));
+  EXPECT_THAT(PacketPayload(packets[1]),
+              ElementsAreArray(FlatVector({StartCode(), {kPps, 4, 5, 6}})));
+  EXPECT_THAT(PacketPayload(packets[2]),
+              ElementsAreArray(FlatVector({StartCode(), {kIdr, 7, 8, 9}})));
+}
+
 TEST(H26xPacketBufferTest, PpsIdrIsNotKeyframeSingleNalus) {
   H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/false);
 
@@ -966,6 +1097,15 @@ TEST(H26xPacketBufferTest, H265IdrIsNotKeyFrame) {
       IsEmpty());
 }
 
+TEST(H26xPacketBufferTest, H265IdrIsNotKeyFrameEvenWithSprop) {
+  H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/true);
+  packet_buffer.SetSpropParameterSets(kExampleSpropString);
+
+  EXPECT_THAT(
+      packet_buffer.InsertPacket(H265Packet().Idr().Marker().Build()).packets,
+      IsEmpty());
+}
+
 TEST(H26xPacketBufferTest, H265SpsPpsIdrIsNotKeyFrame) {
   H26xPacketBuffer packet_buffer(/*h264_allow_idr_only_keyframes=*/false);
 

video/BUILD.gn

@@ -128,6 +128,7 @@ rtc_library("video") {
     "../modules/rtp_rtcp:rtp_rtcp_format",
     "../modules/rtp_rtcp:rtp_video_header",
     "../modules/video_coding",
+    "../modules/video_coding:h264_sprop_parameter_sets",
     "../modules/video_coding:nack_requester",
     "../modules/video_coding:packet_buffer",
     "../modules/video_coding:video_codec_interface",