mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-13 05:40:42 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Make local to capturer clock offset a separate entry in PacketInfo.

Browse source 
This also changes the meaning of |estimated_capture_clock_offset| in
|absolute_capture_time_| to become a remote to capturer clock offset.

Bug: chromium:1056230, webrtc:10739
Change-Id: Id658590e027bbe77ae0834ea224e1dc977a305f2
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/219163
Commit-Queue: Minyue Li <minyue@webrtc.org>
Reviewed-by: Åsa Persson <asapersson@webrtc.org>
Reviewed-by: Erik Språng <sprang@webrtc.org>
Reviewed-by: Alessio Bazzica <alessiob@webrtc.org>
Reviewed-by: Christoffer Rodbro <crodbro@webrtc.org>
Reviewed-by: Gustaf Ullberg <gustaf@webrtc.org>
Reviewed-by: Chen Xing <chxg@google.com>
Cr-Commit-Position: refs/heads/master@{#34067}
This commit is contained in:
Minyue Li 2021-05-19 14:38:25 +02:00 committed by WebRTC LUCI CQ
parent cbde0cf994
commit 63b3095d2b
19 changed files with 557 additions and 350 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
api/rtp_packet_info.cc
View file

@ -71,7 +71,9 @@ bool operator==(const RtpPacketInfo& lhs, const RtpPacketInfo& rhs) {
(lhs.rtp_timestamp() == rhs.rtp_timestamp()) &&
(lhs.audio_level() == rhs.audio_level()) &&
(lhs.absolute_capture_time() == rhs.absolute_capture_time()) &&
(lhs.receive_time() == rhs.receive_time());
(lhs.receive_time() == rhs.receive_time() &&
(lhs.local_capture_clock_offset() ==
rhs.local_capture_clock_offset()));
}
} // namespace webrtc

17
api/rtp_packet_info.h
View file

@ -75,6 +75,14 @@ class RTC_EXPORT RtpPacketInfo {
absolute_capture_time_ = value;
}
const absl::optional<int64_t>& local_capture_clock_offset() const {
return local_capture_clock_offset_;
}
void set_local_capture_clock_offset(const absl::optional<int64_t>& value) {
local_capture_clock_offset_ = value;
}
Timestamp receive_time() const { return receive_time_; }
void set_receive_time(Timestamp value) { receive_time_ = value; }
// TODO(bugs.webrtc.org/12722): Deprecated, remove once downstream projects
@ -94,8 +102,17 @@ class RTC_EXPORT RtpPacketInfo {
// Fields from the Absolute Capture Time header extension:
// http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time
// To not be confused with |local_capture_clock_offset_|, the
// |estimated_capture_clock_offset| in |absolute_capture_time_| should
// represent the clock offset between a remote sender and the capturer, and
// thus equals to the corresponding values in the received RTP packets,
// subjected to possible interpolations.
absl::optional<AbsoluteCaptureTime> absolute_capture_time_;
// Clock offset against capturer's clock. Should be derived from the estimated
// capture clock offset defined in the Absolute Capture Time header extension.
absl::optional<int64_t> local_capture_clock_offset_;
// Local |webrtc::Clock|-based timestamp of when the packet was received.
Timestamp receive_time_;
};

29
api/rtp_packet_info_unittest.cc
View file

@ -149,6 +149,35 @@ TEST(RtpPacketInfoTest, AbsoluteCaptureTime) {
EXPECT_EQ(rhs.absolute_capture_time(), value);
}
TEST(RtpPacketInfoTest, LocalCaptureClockOffset) {
RtpPacketInfo lhs;
RtpPacketInfo rhs;
EXPECT_TRUE(lhs == rhs);
EXPECT_FALSE(lhs != rhs);
const absl::optional<int64_t> value = 10;
rhs.set_local_capture_clock_offset(value);
EXPECT_EQ(rhs.local_capture_clock_offset(), value);
EXPECT_FALSE(lhs == rhs);
EXPECT_TRUE(lhs != rhs);
lhs = rhs;
EXPECT_TRUE(lhs == rhs);
EXPECT_FALSE(lhs != rhs);
// Default local capture clock offset is null.
rhs = RtpPacketInfo();
EXPECT_EQ(rhs.local_capture_clock_offset(), absl::nullopt);
// Default local capture clock offset is null.
rhs = RtpPacketInfo({}, {}, {}, {}, AbsoluteCaptureTime{12, 34},
Timestamp::Millis(0));
EXPECT_EQ(rhs.local_capture_clock_offset(), absl::nullopt);
}
TEST(RtpPacketInfoTest, ReceiveTimeMs) {
const Timestamp timestamp = Timestamp::Micros(8868963877546349045LL);

31
audio/channel_receive.cc
View file

@ -34,7 +34,8 @@
#include "modules/pacing/packet_router.h"
#include "modules/rtp_rtcp/include/receive_statistics.h"
#include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h"
#include "modules/rtp_rtcp/source/absolute_capture_time_receiver.h"
#include "modules/rtp_rtcp/source/absolute_capture_time_interpolator.h"
#include "modules/rtp_rtcp/source/capture_clock_offset_updater.h"
#include "modules/rtp_rtcp/source/rtp_header_extensions.h"
#include "modules/rtp_rtcp/source/rtp_packet_received.h"
#include "modules/rtp_rtcp/source/rtp_rtcp_config.h"
@ -276,7 +277,9 @@ class ChannelReceive : public ChannelReceiveInterface {
rtc::scoped_refptr<FrameDecryptorInterface> frame_decryptor_;
webrtc::CryptoOptions crypto_options_;
webrtc::AbsoluteCaptureTimeReceiver absolute_capture_time_receiver_;
webrtc::AbsoluteCaptureTimeInterpolator absolute_capture_time_interpolator_;
webrtc::CaptureClockOffsetUpdater capture_clock_offset_updater_;
rtc::scoped_refptr<ChannelReceiveFrameTransformerDelegate>
frame_transformer_delegate_;
@ -434,6 +437,22 @@ AudioMixer::Source::AudioFrameInfo ChannelReceive::GetAudioFrameWithInfo(
}
}
// Fill in local capture clock offset in |audio_frame->packet_infos_|.
RtpPacketInfos::vector_type packet_infos;
for (auto& packet_info : audio_frame->packet_infos_) {
absl::optional<int64_t> local_capture_clock_offset;
if (packet_info.absolute_capture_time().has_value()) {
local_capture_clock_offset =
capture_clock_offset_updater_.AdjustEstimatedCaptureClockOffset(
packet_info.absolute_capture_time()
->estimated_capture_clock_offset);
}
RtpPacketInfo new_packet_info(packet_info);
new_packet_info.set_local_capture_clock_offset(local_capture_clock_offset);
packet_infos.push_back(std::move(new_packet_info));
}
audio_frame->packet_infos_ = RtpPacketInfos(packet_infos);
{
RTC_HISTOGRAM_COUNTS_1000("WebRTC.Audio.TargetJitterBufferDelayMs",
acm_receiver_.TargetDelayMs());
@ -502,7 +521,7 @@ ChannelReceive::ChannelReceive(
associated_send_channel_(nullptr),
frame_decryptor_(frame_decryptor),
crypto_options_(crypto_options),
absolute_capture_time_receiver_(clock) {
absolute_capture_time_interpolator_(clock) {
RTC_DCHECK(module_process_thread_);
RTC_DCHECK(audio_device_module);
@ -618,8 +637,8 @@ void ChannelReceive::OnRtpPacket(const RtpPacketReceived& packet) {
// Interpolates absolute capture timestamp RTP header extension.
header.extension.absolute_capture_time =
absolute_capture_time_receiver_.OnReceivePacket(
AbsoluteCaptureTimeReceiver::GetSource(header.ssrc,
absolute_capture_time_interpolator_.OnReceivePacket(
AbsoluteCaptureTimeInterpolator::GetSource(header.ssrc,
header.arrOfCSRCs),
header.timestamp,
rtc::saturated_cast<uint32_t>(packet_copy.payload_type_frequency()),
@ -713,7 +732,7 @@ void ChannelReceive::ReceivedRTCPPacket(const uint8_t* data, size_t length) {
absl::optional<int64_t> remote_to_local_clock_offset_ms =
ntp_estimator_.EstimateRemoteToLocalClockOffsetMs();
if (remote_to_local_clock_offset_ms.has_value()) {
absolute_capture_time_receiver_.SetRemoteToLocalClockOffset(
capture_clock_offset_updater_.SetRemoteToLocalClockOffset(
Int64MsToQ32x32(*remote_to_local_clock_offset_ms));
}
}

11
modules/rtp_rtcp/BUILD.gn
View file

@ -138,12 +138,16 @@ rtc_library("rtp_rtcp") {
"include/receive_statistics.h",
"include/remote_ntp_time_estimator.h",
"include/ulpfec_receiver.h",
"source/absolute_capture_time_receiver.cc",
"source/absolute_capture_time_receiver.h",
"source/absolute_capture_time_interpolator.cc",
"source/absolute_capture_time_interpolator.h",
"source/absolute_capture_time_receiver.cc", # DEPRECATED
"source/absolute_capture_time_receiver.h", # DEPRECATED
"source/absolute_capture_time_sender.cc",
"source/absolute_capture_time_sender.h",
"source/active_decode_targets_helper.cc",
"source/active_decode_targets_helper.h",
"source/capture_clock_offset_updater.cc",
"source/capture_clock_offset_updater.h",
"source/create_video_rtp_depacketizer.cc",
"source/create_video_rtp_depacketizer.h",
"source/dtmf_queue.cc",
@ -473,10 +477,11 @@ if (rtc_include_tests) {
testonly = true
sources = [
"source/absolute_capture_time_receiver_unittest.cc",
"source/absolute_capture_time_interpolator_unittest.cc",
"source/absolute_capture_time_sender_unittest.cc",
"source/active_decode_targets_helper_unittest.cc",
"source/byte_io_unittest.cc",
"source/capture_clock_offset_updater_unittest.cc",
"source/fec_private_tables_bursty_unittest.cc",
"source/flexfec_header_reader_writer_unittest.cc",
"source/flexfec_receiver_unittest.cc",

125
modules/rtp_rtcp/source/absolute_capture_time_interpolator.cc Normal file
View file

@ -0,0 +1,125 @@
/*
* Copyright (c) 2019 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/rtp_rtcp/source/absolute_capture_time_interpolator.h"
#include <limits>
#include "rtc_base/checks.h"
namespace webrtc {
namespace {
constexpr Timestamp kInvalidLastReceiveTime = Timestamp::MinusInfinity();
} // namespace
constexpr TimeDelta AbsoluteCaptureTimeInterpolator::kInterpolationMaxInterval;
AbsoluteCaptureTimeInterpolator::AbsoluteCaptureTimeInterpolator(Clock* clock)
: clock_(clock), last_receive_time_(kInvalidLastReceiveTime) {}
uint32_t AbsoluteCaptureTimeInterpolator::GetSource(
uint32_t ssrc,
rtc::ArrayView<const uint32_t> csrcs) {
if (csrcs.empty()) {
return ssrc;
}
return csrcs[0];
}
absl::optional<AbsoluteCaptureTime>
AbsoluteCaptureTimeInterpolator::OnReceivePacket(
uint32_t source,
uint32_t rtp_timestamp,
uint32_t rtp_clock_frequency,
const absl::optional<AbsoluteCaptureTime>& received_extension) {
const Timestamp receive_time = clock_->CurrentTime();
MutexLock lock(&mutex_);
AbsoluteCaptureTime extension;
if (received_extension == absl::nullopt) {
if (!ShouldInterpolateExtension(receive_time, source, rtp_timestamp,
rtp_clock_frequency)) {
last_receive_time_ = kInvalidLastReceiveTime;
return absl::nullopt;
}
extension.absolute_capture_timestamp = InterpolateAbsoluteCaptureTimestamp(
rtp_timestamp, rtp_clock_frequency, last_rtp_timestamp_,
last_absolute_capture_timestamp_);
extension.estimated_capture_clock_offset =
last_estimated_capture_clock_offset_;
} else {
last_source_ = source;
last_rtp_timestamp_ = rtp_timestamp;
last_rtp_clock_frequency_ = rtp_clock_frequency;
last_absolute_capture_timestamp_ =
received_extension->absolute_capture_timestamp;
last_estimated_capture_clock_offset_ =
received_extension->estimated_capture_clock_offset;
last_receive_time_ = receive_time;
extension = *received_extension;
}
return extension;
}
uint64_t AbsoluteCaptureTimeInterpolator::InterpolateAbsoluteCaptureTimestamp(
uint32_t rtp_timestamp,
uint32_t rtp_clock_frequency,
uint32_t last_rtp_timestamp,
uint64_t last_absolute_capture_timestamp) {
RTC_DCHECK_GT(rtp_clock_frequency, 0);
return last_absolute_capture_timestamp +
static_cast<int64_t>(
rtc::dchecked_cast<uint64_t>(rtp_timestamp - last_rtp_timestamp)
<< 32) /
rtp_clock_frequency;
}
bool AbsoluteCaptureTimeInterpolator::ShouldInterpolateExtension(
Timestamp receive_time,
uint32_t source,
uint32_t rtp_timestamp,
uint32_t rtp_clock_frequency) const {
// Shouldn't if we don't have a previously received extension stored.
if (last_receive_time_ == kInvalidLastReceiveTime) {
return false;
}
// Shouldn't if the last received extension is too old.
if ((receive_time - last_receive_time_) > kInterpolationMaxInterval) {
return false;
}
// Shouldn't if the source has changed.
if (last_source_ != source) {
return false;
}
// Shouldn't if the RTP clock frequency has changed.
if (last_rtp_clock_frequency_ != rtp_clock_frequency) {
return false;
}
// Shouldn't if the RTP clock frequency is invalid.
if (rtp_clock_frequency <= 0) {
return false;
}
return true;
}
} // namespace webrtc

86
modules/rtp_rtcp/source/absolute_capture_time_interpolator.h Normal file
View file

@ -0,0 +1,86 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_RTP_RTCP_SOURCE_ABSOLUTE_CAPTURE_TIME_INTERPOLATOR_H_
#define MODULES_RTP_RTCP_SOURCE_ABSOLUTE_CAPTURE_TIME_INTERPOLATOR_H_
#include "api/array_view.h"
#include "api/rtp_headers.h"
#include "api/units/time_delta.h"
#include "api/units/timestamp.h"
#include "rtc_base/synchronization/mutex.h"
#include "rtc_base/thread_annotations.h"
#include "system_wrappers/include/clock.h"
namespace webrtc {
//
// Helper class for interpolating the |AbsoluteCaptureTime| header extension.
//
// Supports the "timestamp interpolation" optimization:
// A receiver SHOULD memorize the capture system (i.e. CSRC/SSRC), capture
// timestamp, and RTP timestamp of the most recently received abs-capture-time
// packet on each received stream. It can then use that information, in
// combination with RTP timestamps of packets without abs-capture-time, to
// extrapolate missing capture timestamps.
//
// See: https://webrtc.org/experiments/rtp-hdrext/abs-capture-time/
//
class AbsoluteCaptureTimeInterpolator {
public:
static constexpr TimeDelta kInterpolationMaxInterval =
TimeDelta::Millis(5000);
explicit AbsoluteCaptureTimeInterpolator(Clock* clock);
// Returns the source (i.e. SSRC or CSRC) of the capture system.
static uint32_t GetSource(uint32_t ssrc,
rtc::ArrayView<const uint32_t> csrcs);
// Returns a received header extension, an interpolated header extension, or
// |absl::nullopt| if it's not possible to interpolate a header extension.
absl::optional<AbsoluteCaptureTime> OnReceivePacket(
uint32_t source,
uint32_t rtp_timestamp,
uint32_t rtp_clock_frequency,
const absl::optional<AbsoluteCaptureTime>& received_extension);
private:
friend class AbsoluteCaptureTimeSender;
static uint64_t InterpolateAbsoluteCaptureTimestamp(
uint32_t rtp_timestamp,
uint32_t rtp_clock_frequency,
uint32_t last_rtp_timestamp,
uint64_t last_absolute_capture_timestamp);
bool ShouldInterpolateExtension(Timestamp receive_time,
uint32_t source,
uint32_t rtp_timestamp,
uint32_t rtp_clock_frequency) const
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
Clock* const clock_;
Mutex mutex_;
Timestamp last_receive_time_ RTC_GUARDED_BY(mutex_);
uint32_t last_source_ RTC_GUARDED_BY(mutex_);
uint32_t last_rtp_timestamp_ RTC_GUARDED_BY(mutex_);
uint32_t last_rtp_clock_frequency_ RTC_GUARDED_BY(mutex_);
uint64_t last_absolute_capture_timestamp_ RTC_GUARDED_BY(mutex_);
absl::optional<int64_t> last_estimated_capture_clock_offset_
RTC_GUARDED_BY(mutex_);
}; // AbsoluteCaptureTimeInterpolator
} // namespace webrtc
#endif // MODULES_RTP_RTCP_SOURCE_ABSOLUTE_CAPTURE_TIME_INTERPOLATOR_H_

181
modules/rtp_rtcp/source/absolute_capture_time_receiver_unittest.cc → modules/rtp_rtcp/source/absolute_capture_time_interpolator_unittest.cc
View file

@ -8,7 +8,7 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/rtp_rtcp/source/absolute_capture_time_receiver.h"
#include "modules/rtp_rtcp/source/absolute_capture_time_interpolator.h"
#include "system_wrappers/include/ntp_time.h"
#include "test/gmock.h"
@ -16,20 +16,21 @@
namespace webrtc {
TEST(AbsoluteCaptureTimeReceiverTest, GetSourceWithoutCsrcs) {
TEST(AbsoluteCaptureTimeInterpolatorTest, GetSourceWithoutCsrcs) {
constexpr uint32_t kSsrc = 12;
EXPECT_EQ(AbsoluteCaptureTimeReceiver::GetSource(kSsrc, nullptr), kSsrc);
EXPECT_EQ(AbsoluteCaptureTimeInterpolator::GetSource(kSsrc, nullptr), kSsrc);
}
TEST(AbsoluteCaptureTimeReceiverTest, GetSourceWithCsrcs) {
TEST(AbsoluteCaptureTimeInterpolatorTest, GetSourceWithCsrcs) {
constexpr uint32_t kSsrc = 12;
constexpr uint32_t kCsrcs[] = {34, 56, 78, 90};
EXPECT_EQ(AbsoluteCaptureTimeReceiver::GetSource(kSsrc, kCsrcs), kCsrcs[0]);
EXPECT_EQ(AbsoluteCaptureTimeInterpolator::GetSource(kSsrc, kCsrcs),
kCsrcs[0]);
}
TEST(AbsoluteCaptureTimeReceiverTest, ReceiveExtensionReturnsExtension) {
TEST(AbsoluteCaptureTimeInterpolatorTest, ReceiveExtensionReturnsExtension) {
constexpr uint32_t kSource = 1337;
constexpr uint32_t kRtpClockFrequency = 64000;
constexpr uint32_t kRtpTimestamp0 = 1020300000;
@ -40,20 +41,19 @@ TEST(AbsoluteCaptureTimeReceiverTest, ReceiveExtensionReturnsExtension) {
AbsoluteCaptureTime{Int64MsToUQ32x32(9020), absl::nullopt};
SimulatedClock clock(0);
AbsoluteCaptureTimeReceiver receiver(&clock);
AbsoluteCaptureTimeInterpolator interpolator(&clock);
receiver.SetRemoteToLocalClockOffset(0);
EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp0,
EXPECT_EQ(interpolator.OnReceivePacket(kSource, kRtpTimestamp0,
kRtpClockFrequency, kExtension0),
kExtension0);
EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp1,
EXPECT_EQ(interpolator.OnReceivePacket(kSource, kRtpTimestamp1,
kRtpClockFrequency, kExtension1),
kExtension1);
}
TEST(AbsoluteCaptureTimeReceiverTest, ReceiveNoExtensionReturnsNoExtension) {
TEST(AbsoluteCaptureTimeInterpolatorTest,
ReceiveNoExtensionReturnsNoExtension) {
constexpr uint32_t kSource = 1337;
constexpr uint32_t kRtpClockFrequency = 64000;
constexpr uint32_t kRtpTimestamp0 = 1020300000;
@ -62,20 +62,18 @@ TEST(AbsoluteCaptureTimeReceiverTest, ReceiveNoExtensionReturnsNoExtension) {
static const absl::optional<AbsoluteCaptureTime> kExtension1 = absl::nullopt;
SimulatedClock clock(0);
AbsoluteCaptureTimeReceiver receiver(&clock);
AbsoluteCaptureTimeInterpolator interpolator(&clock);
receiver.SetRemoteToLocalClockOffset(0);
EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp0,
EXPECT_EQ(interpolator.OnReceivePacket(kSource, kRtpTimestamp0,
kRtpClockFrequency, kExtension0),
absl::nullopt);
EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp1,
EXPECT_EQ(interpolator.OnReceivePacket(kSource, kRtpTimestamp1,
kRtpClockFrequency, kExtension1),
absl::nullopt);
}
TEST(AbsoluteCaptureTimeReceiverTest, InterpolateLaterPacketArrivingLater) {
TEST(AbsoluteCaptureTimeInterpolatorTest, InterpolateLaterPacketArrivingLater) {
constexpr uint32_t kSource = 1337;
constexpr uint32_t kRtpClockFrequency = 64000;
constexpr uint32_t kRtpTimestamp0 = 1020300000;
@ -87,15 +85,13 @@ TEST(AbsoluteCaptureTimeReceiverTest, InterpolateLaterPacketArrivingLater) {
static const absl::optional<AbsoluteCaptureTime> kExtension2 = absl::nullopt;
SimulatedClock clock(0);
AbsoluteCaptureTimeReceiver receiver(&clock);
AbsoluteCaptureTimeInterpolator interpolator(&clock);
receiver.SetRemoteToLocalClockOffset(0);
EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp0,
EXPECT_EQ(interpolator.OnReceivePacket(kSource, kRtpTimestamp0,
kRtpClockFrequency, kExtension0),
kExtension0);
absl::optional<AbsoluteCaptureTime> extension = receiver.OnReceivePacket(
absl::optional<AbsoluteCaptureTime> extension = interpolator.OnReceivePacket(
kSource, kRtpTimestamp1, kRtpClockFrequency, kExtension1);
EXPECT_TRUE(extension.has_value());
EXPECT_EQ(UQ32x32ToInt64Ms(extension->absolute_capture_timestamp),
@ -103,7 +99,7 @@ TEST(AbsoluteCaptureTimeReceiverTest, InterpolateLaterPacketArrivingLater) {
EXPECT_EQ(extension->estimated_capture_clock_offset,
kExtension0->estimated_capture_clock_offset);
extension = receiver.OnReceivePacket(kSource, kRtpTimestamp2,
extension = interpolator.OnReceivePacket(kSource, kRtpTimestamp2,
kRtpClockFrequency, kExtension2);
EXPECT_TRUE(extension.has_value());
EXPECT_EQ(UQ32x32ToInt64Ms(extension->absolute_capture_timestamp),
@ -112,7 +108,8 @@ TEST(AbsoluteCaptureTimeReceiverTest, InterpolateLaterPacketArrivingLater) {
kExtension0->estimated_capture_clock_offset);
}
TEST(AbsoluteCaptureTimeReceiverTest, InterpolateEarlierPacketArrivingLater) {
TEST(AbsoluteCaptureTimeInterpolatorTest,
InterpolateEarlierPacketArrivingLater) {
constexpr uint32_t kSource = 1337;
constexpr uint32_t kRtpClockFrequency = 64000;
constexpr uint32_t kRtpTimestamp0 = 1020300000;
@ -124,15 +121,13 @@ TEST(AbsoluteCaptureTimeReceiverTest, InterpolateEarlierPacketArrivingLater) {
static const absl::optional<AbsoluteCaptureTime> kExtension2 = absl::nullopt;
SimulatedClock clock(0);
AbsoluteCaptureTimeReceiver receiver(&clock);
AbsoluteCaptureTimeInterpolator interpolator(&clock);
receiver.SetRemoteToLocalClockOffset(0);
EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp0,
EXPECT_EQ(interpolator.OnReceivePacket(kSource, kRtpTimestamp0,
kRtpClockFrequency, kExtension0),
kExtension0);
absl::optional<AbsoluteCaptureTime> extension = receiver.OnReceivePacket(
absl::optional<AbsoluteCaptureTime> extension = interpolator.OnReceivePacket(
kSource, kRtpTimestamp1, kRtpClockFrequency, kExtension1);
EXPECT_TRUE(extension.has_value());
EXPECT_EQ(UQ32x32ToInt64Ms(extension->absolute_capture_timestamp),
@ -140,7 +135,7 @@ TEST(AbsoluteCaptureTimeReceiverTest, InterpolateEarlierPacketArrivingLater) {
EXPECT_EQ(extension->estimated_capture_clock_offset,
kExtension0->estimated_capture_clock_offset);
extension = receiver.OnReceivePacket(kSource, kRtpTimestamp2,
extension = interpolator.OnReceivePacket(kSource, kRtpTimestamp2,
kRtpClockFrequency, kExtension2);
EXPECT_TRUE(extension.has_value());
EXPECT_EQ(UQ32x32ToInt64Ms(extension->absolute_capture_timestamp),
@ -149,7 +144,7 @@ TEST(AbsoluteCaptureTimeReceiverTest, InterpolateEarlierPacketArrivingLater) {
kExtension0->estimated_capture_clock_offset);
}
TEST(AbsoluteCaptureTimeReceiverTest,
TEST(AbsoluteCaptureTimeInterpolatorTest,
InterpolateLaterPacketArrivingLaterWithRtpTimestampWrapAround) {
constexpr uint32_t kSource = 1337;
constexpr uint32_t kRtpClockFrequency = 64000;
@ -162,15 +157,13 @@ TEST(AbsoluteCaptureTimeReceiverTest,
static const absl::optional<AbsoluteCaptureTime> kExtension2 = absl::nullopt;
SimulatedClock clock(0);
AbsoluteCaptureTimeReceiver receiver(&clock);
AbsoluteCaptureTimeInterpolator interpolator(&clock);
receiver.SetRemoteToLocalClockOffset(0);
EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp0,
EXPECT_EQ(interpolator.OnReceivePacket(kSource, kRtpTimestamp0,
kRtpClockFrequency, kExtension0),
kExtension0);
absl::optional<AbsoluteCaptureTime> extension = receiver.OnReceivePacket(
absl::optional<AbsoluteCaptureTime> extension = interpolator.OnReceivePacket(
kSource, kRtpTimestamp1, kRtpClockFrequency, kExtension1);
EXPECT_TRUE(extension.has_value());
EXPECT_EQ(UQ32x32ToInt64Ms(extension->absolute_capture_timestamp),
@ -178,7 +171,7 @@ TEST(AbsoluteCaptureTimeReceiverTest,
EXPECT_EQ(extension->estimated_capture_clock_offset,
kExtension0->estimated_capture_clock_offset);
extension = receiver.OnReceivePacket(kSource, kRtpTimestamp2,
extension = interpolator.OnReceivePacket(kSource, kRtpTimestamp2,
kRtpClockFrequency, kExtension2);
EXPECT_TRUE(extension.has_value());
EXPECT_EQ(UQ32x32ToInt64Ms(extension->absolute_capture_timestamp),
@ -187,7 +180,7 @@ TEST(AbsoluteCaptureTimeReceiverTest,
kExtension0->estimated_capture_clock_offset);
}
TEST(AbsoluteCaptureTimeReceiverTest,
TEST(AbsoluteCaptureTimeInterpolatorTest,
InterpolateEarlierPacketArrivingLaterWithRtpTimestampWrapAround) {
constexpr uint32_t kSource = 1337;
constexpr uint32_t kRtpClockFrequency = 64000;
@ -200,15 +193,13 @@ TEST(AbsoluteCaptureTimeReceiverTest,
static const absl::optional<AbsoluteCaptureTime> kExtension2 = absl::nullopt;
SimulatedClock clock(0);
AbsoluteCaptureTimeReceiver receiver(&clock);
AbsoluteCaptureTimeInterpolator interpolator(&clock);
receiver.SetRemoteToLocalClockOffset(0);
EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp0,
EXPECT_EQ(interpolator.OnReceivePacket(kSource, kRtpTimestamp0,
kRtpClockFrequency, kExtension0),
kExtension0);
absl::optional<AbsoluteCaptureTime> extension = receiver.OnReceivePacket(
absl::optional<AbsoluteCaptureTime> extension = interpolator.OnReceivePacket(
kSource, kRtpTimestamp1, kRtpClockFrequency, kExtension1);
EXPECT_TRUE(extension.has_value());
EXPECT_EQ(UQ32x32ToInt64Ms(extension->absolute_capture_timestamp),
@ -216,7 +207,7 @@ TEST(AbsoluteCaptureTimeReceiverTest,
EXPECT_EQ(extension->estimated_capture_clock_offset,
kExtension0->estimated_capture_clock_offset);
extension = receiver.OnReceivePacket(kSource, kRtpTimestamp2,
extension = interpolator.OnReceivePacket(kSource, kRtpTimestamp2,
kRtpClockFrequency, kExtension2);
EXPECT_TRUE(extension.has_value());
EXPECT_EQ(UQ32x32ToInt64Ms(extension->absolute_capture_timestamp),
@ -225,51 +216,7 @@ TEST(AbsoluteCaptureTimeReceiverTest,
kExtension0->estimated_capture_clock_offset);
}
TEST(AbsoluteCaptureTimeReceiverTest,
SkipEstimatedCaptureClockOffsetIfRemoteToLocalClockOffsetIsUnknown) {
constexpr uint32_t kSource = 1337;
constexpr uint32_t kRtpClockFrequency = 64000;
constexpr uint32_t kRtpTimestamp0 = 1020300000;
constexpr uint32_t kRtpTimestamp1 = kRtpTimestamp0 + 1280;
constexpr uint32_t kRtpTimestamp2 = kRtpTimestamp0 + 2560;
static const absl::optional<AbsoluteCaptureTime> kExtension0 =
AbsoluteCaptureTime{Int64MsToUQ32x32(9000), Int64MsToQ32x32(-350)};
static const absl::optional<AbsoluteCaptureTime> kExtension1 = absl::nullopt;
static const absl::optional<AbsoluteCaptureTime> kExtension2 = absl::nullopt;
static const absl::optional<int64_t> kRemoteToLocalClockOffset2 =
Int64MsToQ32x32(-7000007);
SimulatedClock clock(0);
AbsoluteCaptureTimeReceiver receiver(&clock);
receiver.SetRemoteToLocalClockOffset(0);
EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp0,
kRtpClockFrequency, kExtension0),
kExtension0);
receiver.SetRemoteToLocalClockOffset(absl::nullopt);
absl::optional<AbsoluteCaptureTime> extension = receiver.OnReceivePacket(
kSource, kRtpTimestamp1, kRtpClockFrequency, kExtension1);
EXPECT_TRUE(extension.has_value());
EXPECT_EQ(UQ32x32ToInt64Ms(extension->absolute_capture_timestamp),
UQ32x32ToInt64Ms(kExtension0->absolute_capture_timestamp) + 20);
EXPECT_EQ(extension->estimated_capture_clock_offset, absl::nullopt);
receiver.SetRemoteToLocalClockOffset(kRemoteToLocalClockOffset2);
extension = receiver.OnReceivePacket(kSource, kRtpTimestamp2,
kRtpClockFrequency, kExtension2);
EXPECT_TRUE(extension.has_value());
EXPECT_EQ(UQ32x32ToInt64Ms(extension->absolute_capture_timestamp),
UQ32x32ToInt64Ms(kExtension0->absolute_capture_timestamp) + 40);
EXPECT_EQ(extension->estimated_capture_clock_offset,
*kExtension0->estimated_capture_clock_offset +
*kRemoteToLocalClockOffset2);
}
TEST(AbsoluteCaptureTimeReceiverTest, SkipInterpolateIfTooLate) {
TEST(AbsoluteCaptureTimeInterpolatorTest, SkipInterpolateIfTooLate) {
constexpr uint32_t kSource = 1337;
constexpr uint32_t kRtpClockFrequency = 64000;
constexpr uint32_t kRtpTimestamp0 = 1020300000;
@ -281,30 +228,28 @@ TEST(AbsoluteCaptureTimeReceiverTest, SkipInterpolateIfTooLate) {
static const absl::optional<AbsoluteCaptureTime> kExtension2 = absl::nullopt;
SimulatedClock clock(0);
AbsoluteCaptureTimeReceiver receiver(&clock);
AbsoluteCaptureTimeInterpolator interpolator(&clock);
receiver.SetRemoteToLocalClockOffset(0);
EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp0,
EXPECT_EQ(interpolator.OnReceivePacket(kSource, kRtpTimestamp0,
kRtpClockFrequency, kExtension0),
kExtension0);
clock.AdvanceTime(AbsoluteCaptureTimeReceiver::kInterpolationMaxInterval);
clock.AdvanceTime(AbsoluteCaptureTimeInterpolator::kInterpolationMaxInterval);
EXPECT_TRUE(receiver
EXPECT_TRUE(interpolator
.OnReceivePacket(kSource, kRtpTimestamp1, kRtpClockFrequency,
kExtension1)
.has_value());
clock.AdvanceTimeMilliseconds(1);
EXPECT_FALSE(receiver
EXPECT_FALSE(interpolator
.OnReceivePacket(kSource, kRtpTimestamp2, kRtpClockFrequency,
kExtension2)
.has_value());
}
TEST(AbsoluteCaptureTimeReceiverTest, SkipInterpolateIfSourceChanged) {
TEST(AbsoluteCaptureTimeInterpolatorTest, SkipInterpolateIfSourceChanged) {
constexpr uint32_t kSource0 = 1337;
constexpr uint32_t kSource1 = 1338;
constexpr uint32_t kRtpClockFrequency = 64000;
@ -315,21 +260,19 @@ TEST(AbsoluteCaptureTimeReceiverTest, SkipInterpolateIfSourceChanged) {
static const absl::optional<AbsoluteCaptureTime> kExtension1 = absl::nullopt;
SimulatedClock clock(0);
AbsoluteCaptureTimeReceiver receiver(&clock);
AbsoluteCaptureTimeInterpolator interpolator(&clock);
receiver.SetRemoteToLocalClockOffset(0);
EXPECT_EQ(receiver.OnReceivePacket(kSource0, kRtpTimestamp0,
EXPECT_EQ(interpolator.OnReceivePacket(kSource0, kRtpTimestamp0,
kRtpClockFrequency, kExtension0),
kExtension0);
EXPECT_FALSE(receiver
EXPECT_FALSE(interpolator
.OnReceivePacket(kSource1, kRtpTimestamp1,
kRtpClockFrequency, kExtension1)
.has_value());
}
TEST(AbsoluteCaptureTimeReceiverTest,
TEST(AbsoluteCaptureTimeInterpolatorTest,
SkipInterpolateIfRtpClockFrequencyChanged) {
constexpr uint32_t kSource = 1337;
constexpr uint32_t kRtpClockFrequency0 = 64000;
@ -341,21 +284,19 @@ TEST(AbsoluteCaptureTimeReceiverTest,
static const absl::optional<AbsoluteCaptureTime> kExtension1 = absl::nullopt;
SimulatedClock clock(0);
AbsoluteCaptureTimeReceiver receiver(&clock);
AbsoluteCaptureTimeInterpolator interpolator(&clock);
receiver.SetRemoteToLocalClockOffset(0);
EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp0,
EXPECT_EQ(interpolator.OnReceivePacket(kSource, kRtpTimestamp0,
kRtpClockFrequency0, kExtension0),
kExtension0);
EXPECT_FALSE(receiver
EXPECT_FALSE(interpolator
.OnReceivePacket(kSource, kRtpTimestamp1,
kRtpClockFrequency1, kExtension1)
.has_value());
}
TEST(AbsoluteCaptureTimeReceiverTest,
TEST(AbsoluteCaptureTimeInterpolatorTest,
SkipInterpolateIfRtpClockFrequencyIsInvalid) {
constexpr uint32_t kSource = 1337;
constexpr uint32_t kRtpClockFrequency = 0;
@ -366,21 +307,19 @@ TEST(AbsoluteCaptureTimeReceiverTest,
static const absl::optional<AbsoluteCaptureTime> kExtension1 = absl::nullopt;
SimulatedClock clock(0);
AbsoluteCaptureTimeReceiver receiver(&clock);
AbsoluteCaptureTimeInterpolator interpolator(&clock);
receiver.SetRemoteToLocalClockOffset(0);
EXPECT_EQ(receiver.OnReceivePacket(kSource, kRtpTimestamp0,
EXPECT_EQ(interpolator.OnReceivePacket(kSource, kRtpTimestamp0,
kRtpClockFrequency, kExtension0),
kExtension0);
EXPECT_FALSE(receiver
EXPECT_FALSE(interpolator
.OnReceivePacket(kSource, kRtpTimestamp1, kRtpClockFrequency,
kExtension1)
.has_value());
}
TEST(AbsoluteCaptureTimeReceiverTest, SkipInterpolateIsSticky) {
TEST(AbsoluteCaptureTimeInterpolatorTest, SkipInterpolateIsSticky) {
constexpr uint32_t kSource0 = 1337;
constexpr uint32_t kSource1 = 1338;
constexpr uint32_t kSource2 = 1337;
@ -394,20 +333,18 @@ TEST(AbsoluteCaptureTimeReceiverTest, SkipInterpolateIsSticky) {
static const absl::optional<AbsoluteCaptureTime> kExtension2 = absl::nullopt;
SimulatedClock clock(0);
AbsoluteCaptureTimeReceiver receiver(&clock);
AbsoluteCaptureTimeInterpolator interpolator(&clock);
receiver.SetRemoteToLocalClockOffset(0);
EXPECT_EQ(receiver.OnReceivePacket(kSource0, kRtpTimestamp0,
EXPECT_EQ(interpolator.OnReceivePacket(kSource0, kRtpTimestamp0,
kRtpClockFrequency, kExtension0),
kExtension0);
EXPECT_FALSE(receiver
EXPECT_FALSE(interpolator
.OnReceivePacket(kSource1, kRtpTimestamp1,
kRtpClockFrequency, kExtension1)
.has_value());
EXPECT_FALSE(receiver
EXPECT_FALSE(interpolator
.OnReceivePacket(kSource2, kRtpTimestamp2,
kRtpClockFrequency, kExtension2)
.has_value());

125
modules/rtp_rtcp/source/absolute_capture_time_receiver.cc
View file

@ -10,38 +10,14 @@
#include "modules/rtp_rtcp/source/absolute_capture_time_receiver.h"
#include <limits>
#include "rtc_base/checks.h"
namespace webrtc {
namespace {
constexpr Timestamp kInvalidLastReceiveTime = Timestamp::MinusInfinity();
} // namespace
constexpr TimeDelta AbsoluteCaptureTimeReceiver::kInterpolationMaxInterval;
AbsoluteCaptureTimeReceiver::AbsoluteCaptureTimeReceiver(Clock* clock)
: clock_(clock),
remote_to_local_clock_offset_(absl::nullopt),
last_receive_time_(kInvalidLastReceiveTime) {}
uint32_t AbsoluteCaptureTimeReceiver::GetSource(
uint32_t ssrc,
rtc::ArrayView<const uint32_t> csrcs) {
if (csrcs.empty()) {
return ssrc;
}
return csrcs[0];
}
: AbsoluteCaptureTimeInterpolator(clock) {}
void AbsoluteCaptureTimeReceiver::SetRemoteToLocalClockOffset(
absl::optional<int64_t> value_q32x32) {
MutexLock lock(&mutex_);
remote_to_local_clock_offset_ = value_q32x32;
capture_clock_offset_updater_.SetRemoteToLocalClockOffset(value_q32x32);
}
absl::optional<AbsoluteCaptureTime>
@ -50,101 +26,16 @@ AbsoluteCaptureTimeReceiver::OnReceivePacket(
uint32_t rtp_timestamp,
uint32_t rtp_clock_frequency,
const absl::optional<AbsoluteCaptureTime>& received_extension) {
const Timestamp receive_time = clock_->CurrentTime();
auto extension = AbsoluteCaptureTimeInterpolator::OnReceivePacket(
source, rtp_timestamp, rtp_clock_frequency, received_extension);
MutexLock lock(&mutex_);
AbsoluteCaptureTime extension;
if (received_extension == absl::nullopt) {
if (!ShouldInterpolateExtension(receive_time, source, rtp_timestamp,
rtp_clock_frequency)) {
last_receive_time_ = kInvalidLastReceiveTime;
return absl::nullopt;
if (extension.has_value()) {
extension->estimated_capture_clock_offset =
capture_clock_offset_updater_.AdjustEstimatedCaptureClockOffset(
extension->estimated_capture_clock_offset);
}
extension.absolute_capture_timestamp = InterpolateAbsoluteCaptureTimestamp(
rtp_timestamp, rtp_clock_frequency, last_rtp_timestamp_,
last_absolute_capture_timestamp_);
extension.estimated_capture_clock_offset =
last_estimated_capture_clock_offset_;
} else {
last_source_ = source;
last_rtp_timestamp_ = rtp_timestamp;
last_rtp_clock_frequency_ = rtp_clock_frequency;
last_absolute_capture_timestamp_ =
received_extension->absolute_capture_timestamp;
last_estimated_capture_clock_offset_ =
received_extension->estimated_capture_clock_offset;
last_receive_time_ = receive_time;
extension = *received_extension;
}
extension.estimated_capture_clock_offset = AdjustEstimatedCaptureClockOffset(
extension.estimated_capture_clock_offset);
return extension;
}
uint64_t AbsoluteCaptureTimeReceiver::InterpolateAbsoluteCaptureTimestamp(
uint32_t rtp_timestamp,
uint32_t rtp_clock_frequency,
uint32_t last_rtp_timestamp,
uint64_t last_absolute_capture_timestamp) {
RTC_DCHECK_GT(rtp_clock_frequency, 0);
return last_absolute_capture_timestamp +
static_cast<int64_t>(
rtc::dchecked_cast<uint64_t>(rtp_timestamp - last_rtp_timestamp)
<< 32) /
rtp_clock_frequency;
}
bool AbsoluteCaptureTimeReceiver::ShouldInterpolateExtension(
Timestamp receive_time,
uint32_t source,
uint32_t rtp_timestamp,
uint32_t rtp_clock_frequency) const {
// Shouldn't if we don't have a previously received extension stored.
if (last_receive_time_ == kInvalidLastReceiveTime) {
return false;
}
// Shouldn't if the last received extension is too old.
if ((receive_time - last_receive_time_) > kInterpolationMaxInterval) {
return false;
}
// Shouldn't if the source has changed.
if (last_source_ != source) {
return false;
}
// Shouldn't if the RTP clock frequency has changed.
if (last_rtp_clock_frequency_ != rtp_clock_frequency) {
return false;
}
// Shouldn't if the RTP clock frequency is invalid.
if (rtp_clock_frequency <= 0) {
return false;
}
return true;
}
absl::optional<int64_t>
AbsoluteCaptureTimeReceiver::AdjustEstimatedCaptureClockOffset(
absl::optional<int64_t> received_value) const {
if (received_value == absl::nullopt ||
remote_to_local_clock_offset_ == absl::nullopt) {
return absl::nullopt;
}
// Do calculations as "unsigned" to make overflows deterministic.
return static_cast<uint64_t>(*received_value) +
static_cast<uint64_t>(*remote_to_local_clock_offset_);
}
} // namespace webrtc

77
modules/rtp_rtcp/source/absolute_capture_time_receiver.h
View file

@ -11,89 +11,28 @@
#ifndef MODULES_RTP_RTCP_SOURCE_ABSOLUTE_CAPTURE_TIME_RECEIVER_H_
#define MODULES_RTP_RTCP_SOURCE_ABSOLUTE_CAPTURE_TIME_RECEIVER_H_
#include "api/array_view.h"
#include "api/rtp_headers.h"
#include "api/units/time_delta.h"
#include "api/units/timestamp.h"
#include "rtc_base/synchronization/mutex.h"
#include "rtc_base/thread_annotations.h"
#include "modules/rtp_rtcp/source/absolute_capture_time_interpolator.h"
#include "modules/rtp_rtcp/source/capture_clock_offset_updater.h"
#include "system_wrappers/include/clock.h"
namespace webrtc {
//
// Helper class for receiving the |AbsoluteCaptureTime| header extension.
//
// Supports the "timestamp interpolation" optimization:
// A receiver SHOULD memorize the capture system (i.e. CSRC/SSRC), capture
// timestamp, and RTP timestamp of the most recently received abs-capture-time
// packet on each received stream. It can then use that information, in
// combination with RTP timestamps of packets without abs-capture-time, to
// extrapolate missing capture timestamps.
//
// See: https://webrtc.org/experiments/rtp-hdrext/abs-capture-time/
//
class AbsoluteCaptureTimeReceiver {
// DEPRECATED. Use `AbsoluteCaptureTimeInterpolator` instead.
class AbsoluteCaptureTimeReceiver : public AbsoluteCaptureTimeInterpolator {
public:
static constexpr TimeDelta kInterpolationMaxInterval =
TimeDelta::Millis(5000);
explicit AbsoluteCaptureTimeReceiver(Clock* clock);
// Returns the source (i.e. SSRC or CSRC) of the capture system.
static uint32_t GetSource(uint32_t ssrc,
rtc::ArrayView<const uint32_t> csrcs);
// Sets the NTP clock offset between the sender system (which may be different
// from the capture system) and the local system. This information is normally
// provided by passing half the value of the Round-Trip Time estimation given
// by RTCP sender reports (see DLSR/DLRR).
//
// Note that the value must be in Q32.32-formatted fixed-point seconds.
void SetRemoteToLocalClockOffset(absl::optional<int64_t> value_q32x32);
// Returns a received header extension, an interpolated header extension, or
// |absl::nullopt| if it's not possible to interpolate a header extension.
absl::optional<AbsoluteCaptureTime> OnReceivePacket(
uint32_t source,
uint32_t rtp_timestamp,
uint32_t rtp_clock_frequency,
const absl::optional<AbsoluteCaptureTime>& received_extension);
void SetRemoteToLocalClockOffset(absl::optional<int64_t> value_q32x32);
private:
friend class AbsoluteCaptureTimeSender;
static uint64_t InterpolateAbsoluteCaptureTimestamp(
uint32_t rtp_timestamp,
uint32_t rtp_clock_frequency,
uint32_t last_rtp_timestamp,
uint64_t last_absolute_capture_timestamp);
bool ShouldInterpolateExtension(Timestamp receive_time,
uint32_t source,
uint32_t rtp_timestamp,
uint32_t rtp_clock_frequency) const
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
absl::optional<int64_t> AdjustEstimatedCaptureClockOffset(
absl::optional<int64_t> received_value) const
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_);
Clock* const clock_;
Mutex mutex_;
absl::optional<int64_t> remote_to_local_clock_offset_ RTC_GUARDED_BY(mutex_);
Timestamp last_receive_time_ RTC_GUARDED_BY(mutex_);
uint32_t last_source_ RTC_GUARDED_BY(mutex_);
uint32_t last_rtp_timestamp_ RTC_GUARDED_BY(mutex_);
uint32_t last_rtp_clock_frequency_ RTC_GUARDED_BY(mutex_);
uint64_t last_absolute_capture_timestamp_ RTC_GUARDED_BY(mutex_);
absl::optional<int64_t> last_estimated_capture_clock_offset_
RTC_GUARDED_BY(mutex_);
}; // AbsoluteCaptureTimeReceiver
CaptureClockOffsetUpdater capture_clock_offset_updater_;
};
} // namespace webrtc

33
modules/rtp_rtcp/source/capture_clock_offset_updater.cc Normal file
View file

@ -0,0 +1,33 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/rtp_rtcp/source/capture_clock_offset_updater.h"
namespace webrtc {
absl::optional<int64_t>
CaptureClockOffsetUpdater::AdjustEstimatedCaptureClockOffset(
absl::optional<int64_t> remote_capture_clock_offset) const {
if (remote_capture_clock_offset == absl::nullopt ||
remote_to_local_clock_offset_ == absl::nullopt) {
return absl::nullopt;
}
// Do calculations as "unsigned" to make overflows deterministic.
return static_cast<uint64_t>(*remote_capture_clock_offset) +
static_cast<uint64_t>(*remote_to_local_clock_offset_);
}
void CaptureClockOffsetUpdater::SetRemoteToLocalClockOffset(
absl::optional<int64_t> offset_q32x32) {
remote_to_local_clock_offset_ = offset_q32x32;
}
} // namespace webrtc

51
modules/rtp_rtcp/source/capture_clock_offset_updater.h Normal file
View file

@ -0,0 +1,51 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_RTP_RTCP_SOURCE_CAPTURE_CLOCK_OFFSET_UPDATER_H_
#define MODULES_RTP_RTCP_SOURCE_CAPTURE_CLOCK_OFFSET_UPDATER_H_
#include <stdint.h>
#include "absl/types/optional.h"
namespace webrtc {
//
// Helper class for calculating the clock offset against the capturer's clock.
//
// This is achieved by adjusting the estimated capture clock offset in received
// Absolute Capture Time RTP header extension (see
// https://webrtc.org/experiments/rtp-hdrext/abs-capture-time/), which
// represents the clock offset between a remote sender and the capturer, by
// adding local-to-remote clock offset.
class CaptureClockOffsetUpdater {
public:
// Adjusts remote_capture_clock_offset, which originates from Absolute Capture
// Time RTP header extension, to get the local clock offset against the
// capturer's clock.
absl::optional<int64_t> AdjustEstimatedCaptureClockOffset(
absl::optional<int64_t> remote_capture_clock_offset) const;
// Sets the NTP clock offset between the sender system (which may be different
// from the capture system) and the local system. This information is normally
// provided by passing half the value of the Round-Trip Time estimation given
// by RTCP sender reports (see DLSR/DLRR).
//
// Note that the value must be in Q32.32-formatted fixed-point seconds.
void SetRemoteToLocalClockOffset(absl::optional<int64_t> offset_q32x32);
private:
absl::optional<int64_t> remote_to_local_clock_offset_;
};
} // namespace webrtc
#endif // MODULES_RTP_RTCP_SOURCE_CAPTURE_CLOCK_OFFSET_UPDATER_H_

58
modules/rtp_rtcp/source/capture_clock_offset_updater_unittest.cc Normal file
View file

@ -0,0 +1,58 @@
/*
* Copyright (c) 2021 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/rtp_rtcp/source/capture_clock_offset_updater.h"
#include "system_wrappers/include/ntp_time.h"
#include "test/gmock.h"
#include "test/gtest.h"
namespace webrtc {
TEST(AbsoluteCaptureTimeReceiverTest,
SkipEstimatedCaptureClockOffsetIfRemoteToLocalClockOffsetIsUnknown) {
static const absl::optional<int64_t> kRemoteCaptureClockOffset =
Int64MsToQ32x32(-350);
CaptureClockOffsetUpdater updater;
updater.SetRemoteToLocalClockOffset(absl::nullopt);
EXPECT_EQ(
updater.AdjustEstimatedCaptureClockOffset(kRemoteCaptureClockOffset),
absl::nullopt);
}
TEST(AbsoluteCaptureTimeReceiverTest,
SkipEstimatedCaptureClockOffsetIfRemoteCaptureClockOffsetIsUnknown) {
static const absl::optional<int64_t> kCaptureClockOffsetNull = absl::nullopt;
CaptureClockOffsetUpdater updater;
updater.SetRemoteToLocalClockOffset(0);
EXPECT_EQ(updater.AdjustEstimatedCaptureClockOffset(kCaptureClockOffsetNull),
kCaptureClockOffsetNull);
static const absl::optional<int64_t> kRemoteCaptureClockOffset =
Int64MsToQ32x32(-350);
EXPECT_EQ(
updater.AdjustEstimatedCaptureClockOffset(kRemoteCaptureClockOffset),
kRemoteCaptureClockOffset);
}
TEST(AbsoluteCaptureTimeReceiverTest, EstimatedCaptureClockOffsetArithmetic) {
static const absl::optional<int64_t> kRemoteCaptureClockOffset =
Int64MsToQ32x32(-350);
static const absl::optional<int64_t> kRemoteToLocalClockOffset =
Int64MsToQ32x32(-7000007);
CaptureClockOffsetUpdater updater;
updater.SetRemoteToLocalClockOffset(kRemoteToLocalClockOffset);
EXPECT_THAT(
updater.AdjustEstimatedCaptureClockOffset(kRemoteCaptureClockOffset),
::testing::Optional(::testing::Eq(*kRemoteCaptureClockOffset +
*kRemoteToLocalClockOffset)));
}
} // namespace webrtc

8
video/rtp_video_stream_receiver.cc
View file

@ -274,7 +274,7 @@ RtpVideoStreamReceiver::RtpVideoStreamReceiver(
packet_buffer_(kPacketBufferStartSize, PacketBufferMaxSize()),
has_received_frame_(false),
frames_decryptable_(false),
absolute_capture_time_receiver_(clock) {
absolute_capture_time_interpolator_(clock) {
constexpr bool remb_candidate = true;
if (packet_router_)
packet_router_->AddReceiveRtpModule(rtp_rtcp_.get(), remb_candidate);
@ -655,8 +655,8 @@ void RtpVideoStreamReceiver::OnReceivedPayloadData(
// Try to extrapolate absolute capture time if it is missing.
packet_info.set_absolute_capture_time(
absolute_capture_time_receiver_.OnReceivePacket(
AbsoluteCaptureTimeReceiver::GetSource(packet_info.ssrc(),
absolute_capture_time_interpolator_.OnReceivePacket(
AbsoluteCaptureTimeInterpolator::GetSource(packet_info.ssrc(),
packet_info.csrcs()),
packet_info.rtp_timestamp(),
// Assume frequency is the same one for all video frames.
@ -1114,7 +1114,7 @@ bool RtpVideoStreamReceiver::DeliverRtcp(const uint8_t* rtcp_packet,
absl::optional<int64_t> remote_to_local_clock_offset_ms =
ntp_estimator_.EstimateRemoteToLocalClockOffsetMs();
if (remote_to_local_clock_offset_ms.has_value()) {
absolute_capture_time_receiver_.SetRemoteToLocalClockOffset(
capture_clock_offset_updater_.SetRemoteToLocalClockOffset(
Int64MsToQ32x32(*remote_to_local_clock_offset_ms));
}
}

8
video/rtp_video_stream_receiver.h
View file

@ -33,7 +33,8 @@
#include "modules/rtp_rtcp/include/rtp_header_extension_map.h"
#include "modules/rtp_rtcp/include/rtp_rtcp.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "modules/rtp_rtcp/source/absolute_capture_time_receiver.h"
#include "modules/rtp_rtcp/source/absolute_capture_time_interpolator.h"
#include "modules/rtp_rtcp/source/capture_clock_offset_updater.h"
#include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h"
#include "modules/rtp_rtcp/source/rtp_packet_received.h"
#include "modules/rtp_rtcp/source/rtp_video_header.h"
@ -401,7 +402,10 @@ class RtpVideoStreamReceiver : public LossNotificationSender,
std::atomic<bool> frames_decryptable_;
absl::optional<ColorSpace> last_color_space_;
AbsoluteCaptureTimeReceiver absolute_capture_time_receiver_
AbsoluteCaptureTimeInterpolator absolute_capture_time_interpolator_
RTC_GUARDED_BY(worker_task_checker_);
CaptureClockOffsetUpdater capture_clock_offset_updater_
RTC_GUARDED_BY(worker_task_checker_);
int64_t last_completed_picture_id_ = 0;

8
video/rtp_video_stream_receiver2.cc
View file

@ -253,7 +253,7 @@ RtpVideoStreamReceiver2::RtpVideoStreamReceiver2(
packet_buffer_(kPacketBufferStartSize, PacketBufferMaxSize()),
has_received_frame_(false),
frames_decryptable_(false),
absolute_capture_time_receiver_(clock) {
absolute_capture_time_interpolator_(clock) {
constexpr bool remb_candidate = true;
if (packet_router_)
packet_router_->AddReceiveRtpModule(rtp_rtcp_.get(), remb_candidate);
@ -492,8 +492,8 @@ void RtpVideoStreamReceiver2::OnReceivedPayloadData(
// Try to extrapolate absolute capture time if it is missing.
packet_info.set_absolute_capture_time(
absolute_capture_time_receiver_.OnReceivePacket(
AbsoluteCaptureTimeReceiver::GetSource(packet_info.ssrc(),
absolute_capture_time_interpolator_.OnReceivePacket(
AbsoluteCaptureTimeInterpolator::GetSource(packet_info.ssrc(),
packet_info.csrcs()),
packet_info.rtp_timestamp(),
// Assume frequency is the same one for all video frames.
@ -1036,7 +1036,7 @@ bool RtpVideoStreamReceiver2::DeliverRtcp(const uint8_t* rtcp_packet,
absl::optional<int64_t> remote_to_local_clock_offset_ms =
ntp_estimator_.EstimateRemoteToLocalClockOffsetMs();
if (remote_to_local_clock_offset_ms.has_value()) {
absolute_capture_time_receiver_.SetRemoteToLocalClockOffset(
capture_clock_offset_updater_.SetRemoteToLocalClockOffset(
Int64MsToQ32x32(*remote_to_local_clock_offset_ms));
}
}

8
video/rtp_video_stream_receiver2.h
View file

@ -29,7 +29,8 @@
#include "modules/rtp_rtcp/include/remote_ntp_time_estimator.h"
#include "modules/rtp_rtcp/include/rtp_header_extension_map.h"
#include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
#include "modules/rtp_rtcp/source/absolute_capture_time_receiver.h"
#include "modules/rtp_rtcp/source/absolute_capture_time_interpolator.h"
#include "modules/rtp_rtcp/source/capture_clock_offset_updater.h"
#include "modules/rtp_rtcp/source/rtp_dependency_descriptor_extension.h"
#include "modules/rtp_rtcp/source/rtp_packet_received.h"
#include "modules/rtp_rtcp/source/rtp_rtcp_impl2.h"
@ -350,7 +351,10 @@ class RtpVideoStreamReceiver2 : public LossNotificationSender,
bool frames_decryptable_ RTC_GUARDED_BY(worker_task_checker_);
absl::optional<ColorSpace> last_color_space_;
AbsoluteCaptureTimeReceiver absolute_capture_time_receiver_
AbsoluteCaptureTimeInterpolator absolute_capture_time_interpolator_
RTC_GUARDED_BY(worker_task_checker_);
CaptureClockOffsetUpdater capture_clock_offset_updater_
RTC_GUARDED_BY(worker_task_checker_);
int64_t last_completed_picture_id_ = 0;

4
video/video_receive_stream.cc
View file

@ -506,6 +506,10 @@ void VideoReceiveStream::OnFrame(const VideoFrame& video_frame) {
int64_t video_playout_ntp_ms;
int64_t sync_offset_ms;
double estimated_freq_khz;
// TODO(bugs.webrtc.org/10739): we should set local capture clock offset for
// |video_frame.packet_infos|. But VideoFrame is const qualified here.
// TODO(tommi): GetStreamSyncOffsetInMs grabs three locks. One inside the
// function itself, another in GetChannel() and a third in
// GetPlayoutTimestamp. Seems excessive. Anyhow, I'm assuming the function

3
video/video_receive_stream2.cc
View file

@ -526,6 +526,9 @@ int VideoReceiveStream2::GetBaseMinimumPlayoutDelayMs() const {
void VideoReceiveStream2::OnFrame(const VideoFrame& video_frame) {
VideoFrameMetaData frame_meta(video_frame, clock_->CurrentTime());
// TODO(bugs.webrtc.org/10739): we should set local capture clock offset for
// |video_frame.packet_infos|. But VideoFrame is const qualified here.
worker_thread_->PostTask(
ToQueuedTask(task_safety_, [frame_meta, this]() {
RTC_DCHECK_RUN_ON(&worker_sequence_checker_);