Send and Receive VideoFrameTrackingid RTP header extension.

Bug: webrtc:12594
Change-Id: I2372a361e55d0fdadf9847081644b6a3359a2928
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/212283
Reviewed-by: Artem Titov <titovartem@webrtc.org>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Reviewed-by: Christoffer Rodbro <crodbro@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Commit-Queue: Jeremy Leconte <jleconte@google.com>
Cr-Commit-Position: refs/heads/master@{#33570}

api/rtp_parameters.cc

@@ -130,6 +130,7 @@ constexpr char RtpExtension::kColorSpaceUri[];
 constexpr char RtpExtension::kMidUri[];
 constexpr char RtpExtension::kRidUri[];
 constexpr char RtpExtension::kRepairedRidUri[];
+constexpr char RtpExtension::kVideoFrameTrackingIdUri[];
 
 constexpr int RtpExtension::kMinId;
 constexpr int RtpExtension::kMaxId;
@@ -164,7 +165,8 @@ bool RtpExtension::IsSupportedForVideo(absl::string_view uri) {
          uri == webrtc::RtpExtension::kColorSpaceUri ||
          uri == webrtc::RtpExtension::kRidUri ||
          uri == webrtc::RtpExtension::kRepairedRidUri ||
-         uri == webrtc::RtpExtension::kVideoLayersAllocationUri;
+         uri == webrtc::RtpExtension::kVideoLayersAllocationUri ||
+         uri == webrtc::RtpExtension::kVideoFrameTrackingIdUri;
 }
 
 bool RtpExtension::IsEncryptionSupported(absl::string_view uri) {

api/rtp_parameters.h

@@ -353,6 +353,10 @@ struct RTC_EXPORT RtpExtension {
   static constexpr char kRepairedRidUri[] =
       "urn:ietf:params:rtp-hdrext:sdes:repaired-rtp-stream-id";
 
+  // Header extension to propagate webrtc::VideoFrame id field
+  static constexpr char kVideoFrameTrackingIdUri[] =
+      "http://www.webrtc.org/experiments/rtp-hdrext/video-frame-tracking-id";
+
   // Inclusive min and max IDs for two-byte header extensions and one-byte
   // header extensions, per RFC8285 Section 4.2-4.3.
   static constexpr int kMinId = 1;

api/video/encoded_image.h

@@ -109,6 +109,15 @@ class RTC_EXPORT EncodedImage {
     color_space_ = color_space;
   }
 
+  // These methods along with the private member video_frame_tracking_id_ are
+  // meant for media quality testing purpose only.
+  absl::optional<uint16_t> VideoFrameTrackingId() const {
+    return video_frame_tracking_id_;
+  }
+  void SetVideoFrameTrackingId(absl::optional<uint16_t> tracking_id) {
+    video_frame_tracking_id_ = tracking_id;
+  }
+
   const RtpPacketInfos& PacketInfos() const { return packet_infos_; }
   void SetPacketInfos(RtpPacketInfos packet_infos) {
     packet_infos_ = std::move(packet_infos);
@@ -182,6 +191,9 @@ class RTC_EXPORT EncodedImage {
   absl::optional<int> spatial_index_;
   std::map<int, size_t> spatial_layer_frame_size_bytes_;
   absl::optional<webrtc::ColorSpace> color_space_;
+  // This field is meant for media quality testing purpose only. When enabled it
+  // carries the webrtc::VideoFrame id field from the sender to the receiver.
+  absl::optional<uint16_t> video_frame_tracking_id_;
   // Information about packets used to assemble this video frame. This is needed
   // by |SourceTracker| when the frame is delivered to the RTCRtpReceiver's
   // MediaStreamTrack, in order to implement getContributingSources(). See:

call/rtp_payload_params.cc

@@ -165,6 +165,7 @@ RTPVideoHeader RtpPayloadParams::GetRtpVideoHeader(
   rtp_video_header.color_space = image.ColorSpace()
                                      ? absl::make_optional(*image.ColorSpace())
                                      : absl::nullopt;
+  rtp_video_header.video_frame_tracking_id = image.VideoFrameTrackingId();
   SetVideoTiming(image, &rtp_video_header.video_timing);
 
   const bool is_keyframe = image._frameType == VideoFrameType::kVideoFrameKey;

media/engine/webrtc_video_engine.cc

@@ -686,6 +686,12 @@ WebRtcVideoEngine::GetRtpHeaderExtensions() const {
           ? webrtc::RtpTransceiverDirection::kSendRecv
           : webrtc::RtpTransceiverDirection::kStopped);
 
+  result.emplace_back(
+      webrtc::RtpExtension::kVideoFrameTrackingIdUri, id++,
+      IsEnabled(trials_, "WebRTC-VideoFrameTrackingIdAdvertised")
+          ? webrtc::RtpTransceiverDirection::kSendRecv
+          : webrtc::RtpTransceiverDirection::kStopped);
+
   return result;
 }
 

media/engine/webrtc_video_engine_unittest.cc

@@ -397,6 +397,17 @@ TEST_F(WebRtcVideoEngineTestWithVideoLayersAllocation,
   ExpectRtpCapabilitySupport(RtpExtension::kVideoLayersAllocationUri, true);
 }
 
+class WebRtcVideoFrameTrackingId : public WebRtcVideoEngineTest {
+ public:
+  WebRtcVideoFrameTrackingId()
+      : WebRtcVideoEngineTest(
+            "WebRTC-VideoFrameTrackingIdAdvertised/Enabled/") {}
+};
+
+TEST_F(WebRtcVideoFrameTrackingId, AdvertiseVideoFrameTrackingId) {
+  ExpectRtpCapabilitySupport(RtpExtension::kVideoFrameTrackingIdUri, true);
+}
+
 TEST_F(WebRtcVideoEngineTest, CVOSetHeaderExtensionBeforeCapturer) {
   // Allocate the source first to prevent early destruction before channel's
   // dtor is called.

modules/rtp_rtcp/source/rtp_sender_video.cc

@@ -447,6 +447,11 @@ void RTPSenderVideo::AddRtpHeaderExtensions(
         send_allocation_ == SendVideoLayersAllocation::kSendWithResolution;
     packet->SetExtension<RtpVideoLayersAllocationExtension>(allocation);
   }
+
+  if (first_packet && video_header.video_frame_tracking_id) {
+    packet->SetExtension<VideoFrameTrackingIdExtension>(
+        *video_header.video_frame_tracking_id);
+  }
 }
 
 bool RTPSenderVideo::SendVideo(

modules/rtp_rtcp/source/rtp_video_header.h

@@ -77,6 +77,9 @@ struct RTPVideoHeader {
   VideoPlayoutDelay playout_delay;
   VideoSendTiming video_timing;
   absl::optional<ColorSpace> color_space;
+  // This field is meant for media quality testing purpose only. When enabled it
+  // carries the webrtc::VideoFrame id field from the sender to the receiver.
+  absl::optional<uint16_t> video_frame_tracking_id;
   RTPVideoTypeHeader video_type_header;
 };
 

modules/video_coding/frame_object.cc

@@ -68,6 +68,7 @@ RtpFrameObject::RtpFrameObject(
 
   rotation_ = rotation;
   SetColorSpace(color_space);
+  SetVideoFrameTrackingId(rtp_video_header_.video_frame_tracking_id);
   content_type_ = content_type;
   if (timing.flags != VideoSendTiming::kInvalid) {
     // ntp_time_ms_ may be -1 if not estimated yet. This is not a problem,

test/pc/e2e/analyzer/video/single_process_encoded_image_data_injector.cc

@@ -51,6 +51,7 @@ EncodedImage SingleProcessEncodedImageDataInjector::InjectData(
   buffer->data()[insertion_pos + 2] = info.sub_id;
 
   EncodedImage out = source;
+  out.SetVideoFrameTrackingId(id);
   out.SetEncodedData(buffer);
   return out;
 }

video/rtp_video_stream_receiver.cc

@@ -561,6 +561,8 @@ void RtpVideoStreamReceiver::OnReceivedPayloadData(
       video_header.color_space = last_color_space_;
     }
   }
+  video_header.video_frame_tracking_id =
+      rtp_packet.GetExtension<VideoFrameTrackingIdExtension>();
 
   if (loss_notification_controller_) {
     if (rtp_packet.recovered()) {

video/rtp_video_stream_receiver2.cc

@@ -528,6 +528,8 @@ void RtpVideoStreamReceiver2::OnReceivedPayloadData(
       video_header.color_space = last_color_space_;
     }
   }
+  video_header.video_frame_tracking_id =
+      rtp_packet.GetExtension<VideoFrameTrackingIdExtension>();
 
   if (loss_notification_controller_) {
     if (rtp_packet.recovered()) {