mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-12 21:30:45 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions

Surface local_capture_clock_offset from RtpSource

Browse source 
- Propagating `RtpPacketInfo::local_capture_clock_offset`, an
  existing field that is related to the abs-capture-timestamp
  header extension field `estimated_capture_clock_offset`
- Propagated through `SourceTracker::SourceEntry`

Bug: webrtc:10739, b/246753278
Change-Id: I21d9841e4f3a35da5f8d7b31582898309421d524
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/275241
Reviewed-by: Tomas Gunnarsson <tommi@webrtc.org>
Reviewed-by: Danil Chapovalov <danilchap@webrtc.org>
Commit-Queue: Alessio Bazzica <alessiob@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#38129}
This commit is contained in:
Alessio Bazzica 2022-09-20 11:16:16 +02:00 committed by WebRTC LUCI CQ
parent 54d9f056c6
commit 56b96ffe6a
9 changed files with 123 additions and 40 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
api/BUILD.gn
View file

@ -94,6 +94,7 @@ rtc_library("rtp_packet_info") {
":rtp_headers",
":scoped_refptr",
"../rtc_base/system:rtc_export",
"units:time_delta",
"units:timestamp",
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]

19
api/rtp_packet_info.h
View file

@ -17,6 +17,7 @@
#include "absl/types/optional.h"
#include "api/rtp_headers.h"
#include "api/units/time_delta.h"
#include "api/units/timestamp.h"
#include "rtc_base/system/rtc_export.h"
@ -92,11 +93,11 @@ class RTC_EXPORT RtpPacketInfo {
return *this;
}
const absl::optional<int64_t>& local_capture_clock_offset() const {
const absl::optional<TimeDelta>& local_capture_clock_offset() const {
return local_capture_clock_offset_;
}
RtpPacketInfo& set_local_capture_clock_offset(
const absl::optional<int64_t>& value) {
absl::optional<TimeDelta> value) {
local_capture_clock_offset_ = value;
return *this;
}
@ -117,16 +118,14 @@ class RTC_EXPORT RtpPacketInfo {
// Fields from the Absolute Capture Time header extension:
// http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time
// To not be confused with `local_capture_clock_offset_`, the
// `estimated_capture_clock_offset` in `absolute_capture_time_` should
// represent the clock offset between a remote sender and the capturer, and
// thus equals to the corresponding values in the received RTP packets,
// subjected to possible interpolations.
absl::optional<AbsoluteCaptureTime> absolute_capture_time_;
// Clock offset against capturer's clock. Should be derived from the estimated
// capture clock offset defined in the Absolute Capture Time header extension.
absl::optional<int64_t> local_capture_clock_offset_;
// Clock offset between the local clock and the capturer's clock.
// Do not confuse with `AbsoluteCaptureTime::estimated_capture_clock_offset`
// which instead represents the clock offset between a remote sender and the
// capturer. The following holds:
// Capture's NTP Clock = Local NTP Clock + Local-Capture Clock Offset
absl::optional<TimeDelta> local_capture_clock_offset_;
};
bool operator==(const RtpPacketInfo& lhs, const RtpPacketInfo& rhs);

3
api/rtp_packet_info_unittest.cc
View file

@ -9,6 +9,7 @@
*/
#include "api/rtp_packet_infos.h"
#include "api/units/time_delta.h"
#include "test/gmock.h"
#include "test/gtest.h"
@ -186,7 +187,7 @@ TEST(RtpPacketInfoTest, AbsoluteCaptureTime) {
}
TEST(RtpPacketInfoTest, LocalCaptureClockOffset) {
constexpr absl::optional<int64_t> kValue = 10;
constexpr TimeDelta kValue = TimeDelta::Micros(8868963877546349045LL);
RtpPacketInfo lhs;
RtpPacketInfo rhs;

1
api/transport/rtp/BUILD.gn
View file

@ -13,6 +13,7 @@ rtc_source_set("rtp_source") {
sources = [ "rtp_source.h" ]
deps = [
"../../../api:rtp_headers",
"../../../api/units:time_delta",
"../../../rtc_base:checks",
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]

15
api/transport/rtp/rtp_source.h
View file

@ -15,6 +15,7 @@
#include "absl/types/optional.h"
#include "api/rtp_headers.h"
#include "api/units/time_delta.h"
#include "rtc_base/checks.h"
namespace webrtc {
@ -28,7 +29,17 @@ class RtpSource {
public:
struct Extensions {
absl::optional<uint8_t> audio_level;
// Fields from the Absolute Capture Time header extension:
// http://www.webrtc.org/experiments/rtp-hdrext/abs-capture-time
absl::optional<AbsoluteCaptureTime> absolute_capture_time;
// Clock offset between the local clock and the capturer's clock.
// Do not confuse with `AbsoluteCaptureTime::estimated_capture_clock_offset`
// which instead represents the clock offset between a remote sender and the
// capturer. The following holds:
// Capture's NTP Clock = Local NTP Clock + Local-Capture Clock Offset
absl::optional<TimeDelta> local_capture_clock_offset;
};
RtpSource() = delete;
@ -74,6 +85,10 @@ class RtpSource {
return extensions_.absolute_capture_time;
}
absl::optional<TimeDelta> local_capture_clock_offset() const {
return extensions_.local_capture_clock_offset;
}
bool operator==(const RtpSource& o) const {
return timestamp_ms_ == o.timestamp_ms() && source_id_ == o.source_id() &&
source_type_ == o.source_type() &&

11
audio/channel_receive.cc
View file

@ -23,6 +23,7 @@
#include "api/sequence_checker.h"
#include "api/task_queue/pending_task_safety_flag.h"
#include "api/task_queue/task_queue_base.h"
#include "api/units/time_delta.h"
#include "audio/audio_level.h"
#include "audio/channel_receive_frame_transformer_delegate.h"
#include "audio/channel_send.h"
@ -48,6 +49,7 @@
#include "rtc_base/system/no_unique_address.h"
#include "rtc_base/time_utils.h"
#include "system_wrappers/include/metrics.h"
#include "system_wrappers/include/ntp_time.h"
namespace webrtc {
namespace voe {
@ -464,14 +466,19 @@ AudioMixer::Source::AudioFrameInfo ChannelReceive::GetAudioFrameWithInfo(
// Fill in local capture clock offset in `audio_frame->packet_infos_`.
RtpPacketInfos::vector_type packet_infos;
for (auto& packet_info : audio_frame->packet_infos_) {
absl::optional<int64_t> local_capture_clock_offset;
absl::optional<int64_t> local_capture_clock_offset_q32x32;
if (packet_info.absolute_capture_time().has_value()) {
local_capture_clock_offset =
local_capture_clock_offset_q32x32 =
capture_clock_offset_updater_.AdjustEstimatedCaptureClockOffset(
packet_info.absolute_capture_time()
->estimated_capture_clock_offset);
}
RtpPacketInfo new_packet_info(packet_info);
absl::optional<TimeDelta> local_capture_clock_offset;
if (local_capture_clock_offset_q32x32.has_value()) {
local_capture_clock_offset = TimeDelta::Millis(
UQ32x32ToInt64Ms(*local_capture_clock_offset_q32x32));
}
new_packet_info.set_local_capture_clock_offset(local_capture_clock_offset);
packet_infos.push_back(std::move(new_packet_info));
}

10
modules/rtp_rtcp/source/source_tracker.cc
View file

@ -27,7 +27,7 @@ void SourceTracker::OnFrameDelivered(const RtpPacketInfos& packet_infos) {
int64_t now_ms = clock_->TimeInMilliseconds();
MutexLock lock_scope(&lock_);
for (const auto& packet_info : packet_infos) {
for (const RtpPacketInfo& packet_info : packet_infos) {
for (uint32_t csrc : packet_info.csrcs()) {
SourceKey key(RtpSourceType::CSRC, csrc);
SourceEntry& entry = UpdateEntry(key);
@ -35,6 +35,8 @@ void SourceTracker::OnFrameDelivered(const RtpPacketInfos& packet_infos) {
entry.timestamp_ms = now_ms;
entry.audio_level = packet_info.audio_level();
entry.absolute_capture_time = packet_info.absolute_capture_time();
entry.local_capture_clock_offset =
packet_info.local_capture_clock_offset();
entry.rtp_timestamp = packet_info.rtp_timestamp();
}
@ -44,6 +46,7 @@ void SourceTracker::OnFrameDelivered(const RtpPacketInfos& packet_infos) {
entry.timestamp_ms = now_ms;
entry.audio_level = packet_info.audio_level();
entry.absolute_capture_time = packet_info.absolute_capture_time();
entry.local_capture_clock_offset = packet_info.local_capture_clock_offset();
entry.rtp_timestamp = packet_info.rtp_timestamp();
}
@ -64,7 +67,10 @@ std::vector<RtpSource> SourceTracker::GetSources() const {
sources.emplace_back(
entry.timestamp_ms, key.source, key.source_type, entry.rtp_timestamp,
RtpSource::Extensions{entry.audio_level, entry.absolute_capture_time});
RtpSource::Extensions{
.audio_level = entry.audio_level,
.absolute_capture_time = entry.absolute_capture_time,
.local_capture_clock_offset = entry.local_capture_clock_offset});
}
return sources;

8
modules/rtp_rtcp/source/source_tracker.h
View file

@ -20,6 +20,7 @@
#include "absl/types/optional.h"
#include "api/rtp_packet_infos.h"
#include "api/transport/rtp/rtp_source.h"
#include "api/units/time_delta.h"
#include "rtc_base/synchronization/mutex.h"
#include "rtc_base/time_utils.h"
#include "system_wrappers/include/clock.h"
@ -95,6 +96,13 @@ class SourceTracker {
// https://webrtc.org/experiments/rtp-hdrext/abs-capture-time/
absl::optional<AbsoluteCaptureTime> absolute_capture_time;
// Clock offset between the local clock and the capturer's clock.
// Do not confuse with `AbsoluteCaptureTime::estimated_capture_clock_offset`
// which instead represents the clock offset between a remote sender and the
// capturer. The following holds:
// Capture's NTP Clock = Local NTP Clock + Local-Capture Clock Offset
absl::optional<TimeDelta> local_capture_clock_offset;
// RTP timestamp of the most recent packet used to assemble the frame
// associated with `timestamp_ms`.
uint32_t rtp_timestamp;

95
modules/rtp_rtcp/source/source_tracker_unittest.cc
View file

@ -22,6 +22,7 @@
#include "api/rtp_headers.h"
#include "api/rtp_packet_info.h"
#include "api/rtp_packet_infos.h"
#include "system_wrappers/include/ntp_time.h"
#include "test/gmock.h"
#include "test/gtest.h"
@ -47,8 +48,9 @@ class ExpectedSourceTracker {
const int64_t now_ms = clock_->TimeInMilliseconds();
for (const auto& packet_info : packet_infos) {
RtpSource::Extensions extensions = {packet_info.audio_level(),
packet_info.absolute_capture_time()};
RtpSource::Extensions extensions = {
packet_info.audio_level(), packet_info.absolute_capture_time(),
packet_info.local_capture_clock_offset()};
for (const auto& csrc : packet_info.csrcs()) {
entries_.emplace_front(now_ms, csrc, RtpSourceType::CSRC,
@ -112,7 +114,8 @@ class SourceTrackerRandomTest
.emplace_back(GenerateSsrc(), GenerateCsrcs(), GenerateRtpTimestamp(),
GenerateReceiveTime())
.set_audio_level(GenerateAudioLevel())
.set_absolute_capture_time(GenerateAbsoluteCaptureTime());
.set_absolute_capture_time(GenerateAbsoluteCaptureTime())
.set_local_capture_clock_offset(GenerateLocalCaptureClockOffset());
}
return RtpPacketInfos(std::move(packet_infos));
@ -193,6 +196,14 @@ class SourceTrackerRandomTest
return value;
}
absl::optional<TimeDelta> GenerateLocalCaptureClockOffset() {
if (std::bernoulli_distribution(0.5)(generator_)) {
return absl::nullopt;
}
return TimeDelta::Millis(
UQ32x32ToInt64Ms(std::uniform_int_distribution<int64_t>()(generator_)));
}
Timestamp GenerateReceiveTime() {
return Timestamp::Micros(
std::uniform_int_distribution<int64_t>()(generator_));
@ -254,6 +265,7 @@ TEST(SourceTrackerTest, OnFrameDeliveredRecordsSourcesDistinctSsrcs) {
constexpr absl::optional<AbsoluteCaptureTime> kAbsoluteCaptureTime =
AbsoluteCaptureTime{/*absolute_capture_timestamp=*/12,
/*estimated_capture_clock_offset=*/absl::nullopt};
constexpr absl::optional<TimeDelta> kLocalCaptureClockOffset = absl::nullopt;
constexpr Timestamp kReceiveTime0 = Timestamp::Millis(60);
constexpr Timestamp kReceiveTime1 = Timestamp::Millis(70);
@ -263,18 +275,22 @@ TEST(SourceTrackerTest, OnFrameDeliveredRecordsSourcesDistinctSsrcs) {
tracker.OnFrameDelivered(RtpPacketInfos(
{RtpPacketInfo(kSsrc1, {kCsrcs0, kCsrcs1}, kRtpTimestamp0, kReceiveTime0)
.set_audio_level(kAudioLevel0)
.set_absolute_capture_time(kAbsoluteCaptureTime),
.set_absolute_capture_time(kAbsoluteCaptureTime)
.set_local_capture_clock_offset(kLocalCaptureClockOffset),
RtpPacketInfo(kSsrc2, {kCsrcs2}, kRtpTimestamp1, kReceiveTime1)
.set_audio_level(kAudioLevel1)
.set_absolute_capture_time(kAbsoluteCaptureTime)}));
.set_absolute_capture_time(kAbsoluteCaptureTime)
.set_local_capture_clock_offset(kLocalCaptureClockOffset)}));
int64_t timestamp_ms = clock.TimeInMilliseconds();
constexpr RtpSource::Extensions extensions0 = {
.audio_level = kAudioLevel0,
.absolute_capture_time = kAbsoluteCaptureTime};
.absolute_capture_time = kAbsoluteCaptureTime,
.local_capture_clock_offset = kLocalCaptureClockOffset};
constexpr RtpSource::Extensions extensions1 = {
.audio_level = kAudioLevel1,
.absolute_capture_time = kAbsoluteCaptureTime};
.absolute_capture_time = kAbsoluteCaptureTime,
.local_capture_clock_offset = kLocalCaptureClockOffset};
EXPECT_THAT(tracker.GetSources(),
ElementsAre(RtpSource(timestamp_ms, kSsrc2, RtpSourceType::SSRC,
@ -303,6 +319,7 @@ TEST(SourceTrackerTest, OnFrameDeliveredRecordsSourcesSameSsrc) {
constexpr absl::optional<AbsoluteCaptureTime> kAbsoluteCaptureTime =
AbsoluteCaptureTime{/*absolute_capture_timestamp=*/12,
/*estimated_capture_clock_offset=*/absl::nullopt};
constexpr absl::optional<TimeDelta> kLocalCaptureClockOffset = absl::nullopt;
constexpr Timestamp kReceiveTime0 = Timestamp::Millis(60);
constexpr Timestamp kReceiveTime1 = Timestamp::Millis(70);
constexpr Timestamp kReceiveTime2 = Timestamp::Millis(80);
@ -313,22 +330,31 @@ TEST(SourceTrackerTest, OnFrameDeliveredRecordsSourcesSameSsrc) {
tracker.OnFrameDelivered(RtpPacketInfos({
RtpPacketInfo(kSsrc, {kCsrcs0, kCsrcs1}, kRtpTimestamp0, kReceiveTime0)
.set_audio_level(kAudioLevel0)
.set_absolute_capture_time(kAbsoluteCaptureTime),
.set_absolute_capture_time(kAbsoluteCaptureTime)
.set_local_capture_clock_offset(kLocalCaptureClockOffset),
RtpPacketInfo(kSsrc, {kCsrcs2}, kRtpTimestamp1, kReceiveTime1)
.set_audio_level(kAudioLevel1)
.set_absolute_capture_time(kAbsoluteCaptureTime),
.set_absolute_capture_time(kAbsoluteCaptureTime)
.set_local_capture_clock_offset(kLocalCaptureClockOffset),
RtpPacketInfo(kSsrc, {kCsrcs0}, kRtpTimestamp2, kReceiveTime2)
.set_audio_level(kAudioLevel2)
.set_absolute_capture_time(kAbsoluteCaptureTime),
.set_absolute_capture_time(kAbsoluteCaptureTime)
.set_local_capture_clock_offset(kLocalCaptureClockOffset),
}));
int64_t timestamp_ms = clock.TimeInMilliseconds();
constexpr RtpSource::Extensions extensions0 = {kAudioLevel0,
kAbsoluteCaptureTime};
constexpr RtpSource::Extensions extensions1 = {kAudioLevel1,
kAbsoluteCaptureTime};
constexpr RtpSource::Extensions extensions2 = {kAudioLevel2,
kAbsoluteCaptureTime};
constexpr RtpSource::Extensions extensions0 = {
.audio_level = kAudioLevel0,
.absolute_capture_time = kAbsoluteCaptureTime,
.local_capture_clock_offset = kLocalCaptureClockOffset};
constexpr RtpSource::Extensions extensions1 = {
.audio_level = kAudioLevel1,
.absolute_capture_time = kAbsoluteCaptureTime,
.local_capture_clock_offset = kLocalCaptureClockOffset};
constexpr RtpSource::Extensions extensions2 = {
.audio_level = kAudioLevel2,
.absolute_capture_time = kAbsoluteCaptureTime,
.local_capture_clock_offset = kLocalCaptureClockOffset};
EXPECT_THAT(tracker.GetSources(),
ElementsAre(RtpSource(timestamp_ms, kSsrc, RtpSourceType::SSRC,
@ -359,19 +385,28 @@ TEST(SourceTrackerTest, OnFrameDeliveredUpdatesSources) {
AbsoluteCaptureTime{56, 78};
constexpr absl::optional<AbsoluteCaptureTime> kAbsoluteCaptureTime2 =
AbsoluteCaptureTime{89, 90};
constexpr absl::optional<TimeDelta> kLocalCaptureClockOffset0 =
TimeDelta::Millis(123);
constexpr absl::optional<TimeDelta> kLocalCaptureClockOffset1 =
TimeDelta::Millis(456);
constexpr absl::optional<TimeDelta> kLocalCaptureClockOffset2 =
TimeDelta::Millis(789);
constexpr Timestamp kReceiveTime0 = Timestamp::Millis(60);
constexpr Timestamp kReceiveTime1 = Timestamp::Millis(61);
constexpr Timestamp kReceiveTime2 = Timestamp::Millis(62);
constexpr RtpSource::Extensions extensions0 = {
.audio_level = kAudioLevel0,
.absolute_capture_time = kAbsoluteCaptureTime0};
.absolute_capture_time = kAbsoluteCaptureTime0,
.local_capture_clock_offset = kLocalCaptureClockOffset0};
constexpr RtpSource::Extensions extensions1 = {
.audio_level = kAudioLevel1,
.absolute_capture_time = kAbsoluteCaptureTime1};
.absolute_capture_time = kAbsoluteCaptureTime1,
.local_capture_clock_offset = kLocalCaptureClockOffset1};
constexpr RtpSource::Extensions extensions2 = {
.audio_level = kAudioLevel2,
.absolute_capture_time = kAbsoluteCaptureTime2};
.absolute_capture_time = kAbsoluteCaptureTime2,
.local_capture_clock_offset = kLocalCaptureClockOffset2};
SimulatedClock clock(1000000000000ULL);
SourceTracker tracker(&clock);
@ -379,7 +414,8 @@ TEST(SourceTrackerTest, OnFrameDeliveredUpdatesSources) {
tracker.OnFrameDelivered(RtpPacketInfos(
{RtpPacketInfo(kSsrc1, {kCsrcs0, kCsrcs1}, kRtpTimestamp0, kReceiveTime0)
.set_audio_level(kAudioLevel0)
.set_absolute_capture_time(kAbsoluteCaptureTime0)}));
.set_absolute_capture_time(kAbsoluteCaptureTime0)
.set_local_capture_clock_offset(kLocalCaptureClockOffset0)}));
int64_t timestamp_ms_0 = clock.TimeInMilliseconds();
EXPECT_THAT(
@ -397,7 +433,8 @@ TEST(SourceTrackerTest, OnFrameDeliveredUpdatesSources) {
tracker.OnFrameDelivered(RtpPacketInfos(
{RtpPacketInfo(kSsrc1, {kCsrcs0, kCsrcs2}, kRtpTimestamp1, kReceiveTime1)
.set_audio_level(kAudioLevel1)
.set_absolute_capture_time(kAbsoluteCaptureTime1)}));
.set_absolute_capture_time(kAbsoluteCaptureTime1)
.set_local_capture_clock_offset(kLocalCaptureClockOffset1)}));
int64_t timestamp_ms_1 = clock.TimeInMilliseconds();
@ -418,7 +455,8 @@ TEST(SourceTrackerTest, OnFrameDeliveredUpdatesSources) {
tracker.OnFrameDelivered(RtpPacketInfos(
{RtpPacketInfo(kSsrc2, {kCsrcs0}, kRtpTimestamp2, kReceiveTime2)
.set_audio_level(kAudioLevel2)
.set_absolute_capture_time(kAbsoluteCaptureTime2)}));
.set_absolute_capture_time(kAbsoluteCaptureTime2)
.set_local_capture_clock_offset(kLocalCaptureClockOffset2)}));
int64_t timestamp_ms_2 = clock.TimeInMilliseconds();
@ -449,6 +487,10 @@ TEST(SourceTrackerTest, TimedOutSourcesAreRemoved) {
AbsoluteCaptureTime{12, 34};
constexpr absl::optional<AbsoluteCaptureTime> kAbsoluteCaptureTime1 =
AbsoluteCaptureTime{56, 78};
constexpr absl::optional<TimeDelta> kLocalCaptureClockOffset0 =
TimeDelta::Millis(123);
constexpr absl::optional<TimeDelta> kLocalCaptureClockOffset1 =
TimeDelta::Millis(456);
constexpr Timestamp kReceiveTime0 = Timestamp::Millis(60);
constexpr Timestamp kReceiveTime1 = Timestamp::Millis(61);
@ -458,14 +500,16 @@ TEST(SourceTrackerTest, TimedOutSourcesAreRemoved) {
tracker.OnFrameDelivered(RtpPacketInfos(
{RtpPacketInfo(kSsrc, {kCsrcs0, kCsrcs1}, kRtpTimestamp0, kReceiveTime0)
.set_audio_level(kAudioLevel0)
.set_absolute_capture_time(kAbsoluteCaptureTime0)}));
.set_absolute_capture_time(kAbsoluteCaptureTime0)
.set_local_capture_clock_offset(kLocalCaptureClockOffset0)}));
clock.AdvanceTimeMilliseconds(17);
tracker.OnFrameDelivered(RtpPacketInfos(
{RtpPacketInfo(kSsrc, {kCsrcs0, kCsrcs2}, kRtpTimestamp1, kReceiveTime1)
.set_audio_level(kAudioLevel1)
.set_absolute_capture_time(kAbsoluteCaptureTime1)}));
.set_absolute_capture_time(kAbsoluteCaptureTime1)
.set_local_capture_clock_offset(kLocalCaptureClockOffset1)}));
int64_t timestamp_ms_1 = clock.TimeInMilliseconds();
@ -473,7 +517,8 @@ TEST(SourceTrackerTest, TimedOutSourcesAreRemoved) {
constexpr RtpSource::Extensions extensions1 = {
.audio_level = kAudioLevel1,
.absolute_capture_time = kAbsoluteCaptureTime1};
.absolute_capture_time = kAbsoluteCaptureTime1,
.local_capture_clock_offset = kLocalCaptureClockOffset1};
EXPECT_THAT(
tracker.GetSources(),