Handling NetEqSetMinimumDelay events in neteq_rtpplay.

Bug: webrtc:14763
Change-Id: I81a832209249468f8cec682b13bd025a1cec47b8
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/291322
Reviewed-by: Jakob Ivarsson‎ <jakobi@webrtc.org>
Reviewed-by: Sam Zackrisson <saza@webrtc.org>
Reviewed-by: Lionel Koenig <lionelk@webrtc.org>
Reviewed-by: Björn Terelius <terelius@webrtc.org>
Commit-Queue: Jesus de Vicente Pena <devicentepena@webrtc.org>
Cr-Commit-Position: refs/heads/main@{#39280}

modules/audio_coding/neteq/neteq_decoder_plc_unittest.cc

@@ -127,6 +127,10 @@ class LossyInput : public NetEqInput {
     return input_->NextOutputEventTime();
   }
 
+  absl::optional<SetMinimumDelayInfo> NextSetMinimumDelayInfo() const override {
+    return input_->NextSetMinimumDelayInfo();
+  }
+
   std::unique_ptr<PacketData> PopPacket() override {
     if (loss_cadence_ != 0 && (++count_ % loss_cadence_) == 0) {
       // Pop `burst_length_` packets to create the loss.
@@ -141,6 +145,10 @@ class LossyInput : public NetEqInput {
 
   void AdvanceOutputEvent() override { return input_->AdvanceOutputEvent(); }
 
+  void AdvanceSetMinimumDelay() override {
+    return input_->AdvanceSetMinimumDelay();
+  }
+
   bool ended() const override { return input_->ended(); }
 
   absl::optional<RTPHeader> NextHeader() const override {

modules/audio_coding/neteq/tools/encode_neteq_input.h

@@ -41,10 +41,16 @@ class EncodeNetEqInput : public NetEqInput {
 
   absl::optional<int64_t> NextOutputEventTime() const override;
 
+  absl::optional<SetMinimumDelayInfo> NextSetMinimumDelayInfo() const override {
+    return absl::nullopt;
+  }
+
   std::unique_ptr<PacketData> PopPacket() override;
 
   void AdvanceOutputEvent() override;
 
+  void AdvanceSetMinimumDelay() override {}
+
   bool ended() const override;
 
   absl::optional<RTPHeader> NextHeader() const override;

modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.cc

@@ -37,6 +37,11 @@ absl::optional<int64_t> InitialPacketInserterNetEqInput::NextOutputEventTime()
   return source_->NextOutputEventTime();
 }
 
+absl::optional<NetEqInput::SetMinimumDelayInfo>
+InitialPacketInserterNetEqInput::NextSetMinimumDelayInfo() const {
+  return source_->NextSetMinimumDelayInfo();
+}
+
 std::unique_ptr<InitialPacketInserterNetEqInput::PacketData>
 InitialPacketInserterNetEqInput::PopPacket() {
   if (!first_packet_) {
@@ -63,6 +68,10 @@ InitialPacketInserterNetEqInput::PopPacket() {
   return source_->PopPacket();
 }
 
+void InitialPacketInserterNetEqInput::AdvanceSetMinimumDelay() {
+  source_->AdvanceSetMinimumDelay();
+}
+
 void InitialPacketInserterNetEqInput::AdvanceOutputEvent() {
   source_->AdvanceOutputEvent();
 }

modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.h

@@ -29,8 +29,10 @@ class InitialPacketInserterNetEqInput final : public NetEqInput {
                                   int sample_rate_hz);
   absl::optional<int64_t> NextPacketTime() const override;
   absl::optional<int64_t> NextOutputEventTime() const override;
+  absl::optional<SetMinimumDelayInfo> NextSetMinimumDelayInfo() const override;
   std::unique_ptr<PacketData> PopPacket() override;
   void AdvanceOutputEvent() override;
+  void AdvanceSetMinimumDelay() override;
   bool ended() const override;
   absl::optional<RTPHeader> NextHeader() const override;
 

modules/audio_coding/neteq/tools/neteq_event_log_input.cc

@@ -46,6 +46,11 @@ absl::optional<int64_t> NetEqEventLogInput::NextOutputEventTime() const {
   return next_output_event_ms_;
 }
 
+absl::optional<NetEqInput::SetMinimumDelayInfo>
+NetEqEventLogInput::NextSetMinimumDelayInfo() const {
+  return next_minimum_delay_event_info_;
+}
+
 void NetEqEventLogInput::AdvanceOutputEvent() {
   next_output_event_ms_ = source_->NextAudioOutputEventMs();
   if (*next_output_event_ms_ == std::numeric_limits<int64_t>::max()) {
@@ -53,6 +58,10 @@ void NetEqEventLogInput::AdvanceOutputEvent() {
   }
 }
 
+void NetEqEventLogInput::AdvanceSetMinimumDelay() {
+  next_minimum_delay_event_info_ = source_->NextSetMinimumDelayEvent();
+}
+
 PacketSource* NetEqEventLogInput::source() {
   return source_.get();
 }
@@ -62,6 +71,7 @@ NetEqEventLogInput::NetEqEventLogInput(
     : source_(std::move(source)) {
   LoadNextPacket();
   AdvanceOutputEvent();
+  AdvanceSetMinimumDelay();
 }
 
 }  // namespace test

modules/audio_coding/neteq/tools/neteq_event_log_input.h

@@ -36,7 +36,9 @@ class NetEqEventLogInput final : public NetEqPacketSourceInput {
       absl::optional<uint32_t> ssrc_filter);
 
   absl::optional<int64_t> NextOutputEventTime() const override;
+  absl::optional<SetMinimumDelayInfo> NextSetMinimumDelayInfo() const override;
   void AdvanceOutputEvent() override;
+  void AdvanceSetMinimumDelay() override;
 
  protected:
   PacketSource* source() override;
@@ -44,6 +46,7 @@ class NetEqEventLogInput final : public NetEqPacketSourceInput {
  private:
   NetEqEventLogInput(std::unique_ptr<RtcEventLogSource> source);
   std::unique_ptr<RtcEventLogSource> source_;
+  absl::optional<SetMinimumDelayInfo> next_minimum_delay_event_info_;
 };
 
 }  // namespace test

modules/audio_coding/neteq/tools/neteq_input.cc

@@ -58,6 +58,11 @@ absl::optional<int64_t> TimeLimitedNetEqInput::NextOutputEventTime() const {
   return ended_ ? absl::nullopt : input_->NextOutputEventTime();
 }
 
+absl::optional<NetEqInput::SetMinimumDelayInfo>
+TimeLimitedNetEqInput::NextSetMinimumDelayInfo() const {
+  return ended_ ? absl::nullopt : input_->NextSetMinimumDelayInfo();
+}
+
 std::unique_ptr<NetEqInput::PacketData> TimeLimitedNetEqInput::PopPacket() {
   if (ended_) {
     return std::unique_ptr<PacketData>();
@@ -74,6 +79,13 @@ void TimeLimitedNetEqInput::AdvanceOutputEvent() {
   }
 }
 
+void TimeLimitedNetEqInput::AdvanceSetMinimumDelay() {
+  if (!ended_) {
+    input_->AdvanceSetMinimumDelay();
+    MaybeSetEnded();
+  }
+}
+
 bool TimeLimitedNetEqInput::ended() const {
   return ended_ || input_->ended();
 }

modules/audio_coding/neteq/tools/neteq_input.h

@@ -36,6 +36,13 @@ class NetEqInput {
     int64_t time_ms;
   };
 
+  struct SetMinimumDelayInfo {
+    SetMinimumDelayInfo(int64_t timestamp_ms_in, int delay_ms_in)
+        : timestamp_ms(timestamp_ms_in), delay_ms(delay_ms_in) {}
+    int64_t timestamp_ms;
+    int delay_ms;
+  };
+
   virtual ~NetEqInput() = default;
 
   // Returns at what time (in ms) NetEq::InsertPacket should be called next, or
@@ -46,16 +53,31 @@ class NetEqInput {
   // empty if no more output events are available.
   virtual absl::optional<int64_t> NextOutputEventTime() const = 0;
 
-  // Returns the time (in ms) for the next event from either NextPacketTime()
+  // Returns the information related to the next NetEq set minimum delay event
-  // or NextOutputEventTime(), or empty if both are out of events.
+  // if available.
+  virtual absl::optional<SetMinimumDelayInfo> NextSetMinimumDelayInfo()
+      const = 0;
+
+  // Returns the time (in ms) for the next event (packet, output or set minimum
+  // delay event) or empty if there are no more events.
   absl::optional<int64_t> NextEventTime() const {
-    const auto a = NextPacketTime();
+    absl::optional<int64_t> next_event_time = NextPacketTime();
-    const auto b = NextOutputEventTime();
+    const auto next_output_time = NextOutputEventTime();
     // Return the minimum of non-empty `a` and `b`, or empty if both are empty.
-    if (a) {
+    if (next_output_time) {
-      return b ? std::min(*a, *b) : a;
+      next_event_time = next_event_time ? std::min(next_event_time.value(),
+                                                   next_output_time.value())
+                                        : next_output_time;
     }
-    return b ? b : absl::nullopt;
+    const auto next_neteq_minimum_delay = NextSetMinimumDelayInfo();
+    if (next_neteq_minimum_delay) {
+      next_event_time =
+          next_event_time
+              ? std::min(next_event_time.value(),
+                         next_neteq_minimum_delay.value().timestamp_ms)
+              : next_neteq_minimum_delay.value().timestamp_ms;
+    }
+    return next_event_time;
   }
 
   // Returns the next packet to be inserted into NetEq. The packet following the
@@ -69,6 +91,10 @@ class NetEqInput {
   // time).
   virtual void AdvanceOutputEvent() = 0;
 
+  // Move to the next NetEq set minimum delay. This will make
+  // `NextSetMinimumDelayInfo` return a new value.
+  virtual void AdvanceSetMinimumDelay() = 0;
+
   // Returns true if the source has come to an end. An implementation must
   // eventually return true from this method, or the test will end up in an
   // infinite loop.
@@ -88,8 +114,10 @@ class TimeLimitedNetEqInput : public NetEqInput {
   ~TimeLimitedNetEqInput() override;
   absl::optional<int64_t> NextPacketTime() const override;
   absl::optional<int64_t> NextOutputEventTime() const override;
+  absl::optional<SetMinimumDelayInfo> NextSetMinimumDelayInfo() const override;
   std::unique_ptr<PacketData> PopPacket() override;
   void AdvanceOutputEvent() override;
+  void AdvanceSetMinimumDelay() override;
   bool ended() const override;
   absl::optional<RTPHeader> NextHeader() const override;
 

modules/audio_coding/neteq/tools/neteq_packet_source_input.h

@@ -54,7 +54,11 @@ class NetEqRtpDumpInput final : public NetEqPacketSourceInput {
                     absl::optional<uint32_t> ssrc_filter);
 
   absl::optional<int64_t> NextOutputEventTime() const override;
+  absl::optional<SetMinimumDelayInfo> NextSetMinimumDelayInfo() const override {
+    return absl::nullopt;
+  }
   void AdvanceOutputEvent() override;
+  void AdvanceSetMinimumDelay() override {}
 
  protected:
   PacketSource* source() override;

modules/audio_coding/neteq/tools/neteq_replacement_input.cc

@@ -40,6 +40,11 @@ absl::optional<int64_t> NetEqReplacementInput::NextOutputEventTime() const {
   return source_->NextOutputEventTime();
 }
 
+absl::optional<NetEqInput::SetMinimumDelayInfo>
+NetEqReplacementInput::NextSetMinimumDelayInfo() const {
+  return source_->NextSetMinimumDelayInfo();
+}
+
 std::unique_ptr<NetEqInput::PacketData> NetEqReplacementInput::PopPacket() {
   std::unique_ptr<PacketData> to_return = std::move(packet_);
   while (true) {
@@ -59,6 +64,10 @@ void NetEqReplacementInput::AdvanceOutputEvent() {
   source_->AdvanceOutputEvent();
 }
 
+void NetEqReplacementInput::AdvanceSetMinimumDelay() {
+  source_->AdvanceSetMinimumDelay();
+}
+
 bool NetEqReplacementInput::ended() const {
   return source_->ended();
 }

modules/audio_coding/neteq/tools/neteq_replacement_input.h

@@ -30,8 +30,10 @@ class NetEqReplacementInput : public NetEqInput {
 
   absl::optional<int64_t> NextPacketTime() const override;
   absl::optional<int64_t> NextOutputEventTime() const override;
+  absl::optional<SetMinimumDelayInfo> NextSetMinimumDelayInfo() const override;
   std::unique_ptr<PacketData> PopPacket() override;
   void AdvanceOutputEvent() override;
+  void AdvanceSetMinimumDelay() override;
   bool ended() const override;
   absl::optional<RTPHeader> NextHeader() const override;
 

modules/audio_coding/neteq/tools/neteq_test.cc

@@ -163,6 +163,13 @@ NetEqTest::SimulationStepResult NetEqTest::RunToNextGetAudio() {
           absl::make_optional<uint32_t>(packet_data->header.timestamp);
     }
 
+    if (input_->NextSetMinimumDelayInfo().has_value() &&
+        time_now_ms >= input_->NextSetMinimumDelayInfo().value().timestamp_ms) {
+      neteq_->SetBaseMinimumDelayMs(
+          input_->NextSetMinimumDelayInfo().value().delay_ms);
+      input_->AdvanceSetMinimumDelay();
+    }
+
     // Check if it is time to get output audio.
     if (input_->NextOutputEventTime() &&
         time_now_ms >= *input_->NextOutputEventTime()) {

modules/audio_coding/neteq/tools/rtc_event_log_source.cc

@@ -20,6 +20,7 @@
 
 #include "absl/strings/string_view.h"
 #include "logging/rtc_event_log/rtc_event_processor.h"
+#include "modules/audio_coding/neteq/tools/neteq_input.h"
 #include "modules/audio_coding/neteq/tools/packet.h"
 #include "rtc_base/checks.h"
 
@@ -94,6 +95,14 @@ int64_t RtcEventLogSource::NextAudioOutputEventMs() {
   return output_time_ms;
 }
 
+absl::optional<NetEqInput::SetMinimumDelayInfo>
+RtcEventLogSource::NextSetMinimumDelayEvent() {
+  if (minimum_delay_index_ >= minimum_delay_.size()) {
+    return absl::nullopt;
+  }
+  return minimum_delay_[minimum_delay_index_++];
+}
+
 RtcEventLogSource::RtcEventLogSource() : PacketSource() {}
 
 bool RtcEventLogSource::Initialize(const ParsedRtcEventLog& parsed_log,
@@ -130,6 +139,17 @@ bool RtcEventLogSource::Initialize(const ParsedRtcEventLog& parsed_log,
         }
       };
 
+  auto handle_neteq_set_minimum_delay =
+      [this, first_log_end_time_us, &packet_ssrcs](
+          const webrtc::LoggedNetEqSetMinimumDelayEvent minimum_delay_event) {
+        if (minimum_delay_event.log_time_us() < first_log_end_time_us) {
+          if (packet_ssrcs.count(minimum_delay_event.remote_ssrc) > 0) {
+            minimum_delay_.emplace_back(minimum_delay_event.log_time_ms(),
+                                        minimum_delay_event.minimum_delay_ms);
+          }
+        }
+      };
+
   // This wouldn't be needed if we knew that there was at most one audio stream.
   webrtc::RtcEventProcessor event_processor;
   for (const auto& rtp_packets : parsed_log.incoming_rtp_packets_by_ssrc()) {
@@ -152,6 +172,16 @@ bool RtcEventLogSource::Initialize(const ParsedRtcEventLog& parsed_log,
     event_processor.AddEvents(audio_playouts.second, handle_audio_playout);
   }
 
+  for (const auto& neteq_set_minimum_delay_event :
+       parsed_log.neteq_set_minimum_delay_events()) {
+    if (ssrc_filter.has_value() &&
+        neteq_set_minimum_delay_event.first != *ssrc_filter) {
+      continue;
+    }
+    event_processor.AddEvents(neteq_set_minimum_delay_event.second,
+                              handle_neteq_set_minimum_delay);
+  }
+
   // Fills in rtp_packets_ and audio_outputs_.
   event_processor.ProcessEventsInOrder();
 

modules/audio_coding/neteq/tools/rtc_event_log_source.h

@@ -18,6 +18,7 @@
 #include "absl/strings/string_view.h"
 #include "absl/types/optional.h"
 #include "logging/rtc_event_log/rtc_event_log_parser.h"
+#include "modules/audio_coding/neteq/tools/neteq_input.h"
 #include "modules/audio_coding/neteq/tools/packet_source.h"
 #include "modules/rtp_rtcp/include/rtp_rtcp_defines.h"
 
@@ -53,6 +54,9 @@ class RtcEventLogSource : public PacketSource {
   // events available.
   int64_t NextAudioOutputEventMs();
 
+  // Returns the next NetEq set minimum delay event if available.
+  absl::optional<NetEqInput::SetMinimumDelayInfo> NextSetMinimumDelayEvent();
+
  private:
   RtcEventLogSource();
 
@@ -63,6 +67,8 @@ class RtcEventLogSource : public PacketSource {
   size_t rtp_packet_index_ = 0;
   std::vector<int64_t> audio_outputs_;
   size_t audio_output_index_ = 0;
+  std::vector<NetEqInput::SetMinimumDelayInfo> minimum_delay_;
+  size_t minimum_delay_index_ = 0;
 };
 
 }  // namespace test

rtc_tools/rtc_event_log_visualizer/analyze_audio.cc

@@ -182,11 +182,17 @@ class NetEqStreamInput : public test::NetEqInput {
   // that outlive the one constructed.
   NetEqStreamInput(const std::vector<LoggedRtpPacketIncoming>* packet_stream,
                    const std::vector<LoggedAudioPlayoutEvent>* output_events,
+                   const std::vector<LoggedNetEqSetMinimumDelayEvent>*
+                       neteq_set_minimum_delay_events,
                    absl::optional<int64_t> end_time_ms)
       : packet_stream_(*packet_stream),
         packet_stream_it_(packet_stream_.begin()),
         output_events_it_(output_events->begin()),
         output_events_end_(output_events->end()),
+        neteq_set_minimum_delay_events_it_(
+            neteq_set_minimum_delay_events->begin()),
+        neteq_set_minimum_delay_events_end_(
+            neteq_set_minimum_delay_events->end()),
         end_time_ms_(end_time_ms) {
     RTC_DCHECK(packet_stream);
     RTC_DCHECK(output_events);
@@ -212,6 +218,20 @@ class NetEqStreamInput : public test::NetEqInput {
     return output_events_it_->log_time_ms();
   }
 
+  absl::optional<SetMinimumDelayInfo> NextSetMinimumDelayInfo() const override {
+    if (neteq_set_minimum_delay_events_it_ ==
+        neteq_set_minimum_delay_events_end_) {
+      return absl::nullopt;
+    }
+    if (end_time_ms_ &&
+        neteq_set_minimum_delay_events_it_->log_time_ms() > *end_time_ms_) {
+      return absl::nullopt;
+    }
+    return SetMinimumDelayInfo(
+        neteq_set_minimum_delay_events_it_->log_time_ms(),
+        neteq_set_minimum_delay_events_it_->minimum_delay_ms);
+  }
+
   std::unique_ptr<PacketData> PopPacket() override {
     if (packet_stream_it_ == packet_stream_.end()) {
       return std::unique_ptr<PacketData>();
@@ -236,6 +256,13 @@ class NetEqStreamInput : public test::NetEqInput {
     }
   }
 
+  void AdvanceSetMinimumDelay() override {
+    if (neteq_set_minimum_delay_events_it_ !=
+        neteq_set_minimum_delay_events_end_) {
+      ++neteq_set_minimum_delay_events_it_;
+    }
+  }
+
   bool ended() const override { return !NextEventTime(); }
 
   absl::optional<RTPHeader> NextHeader() const override {
@@ -250,6 +277,10 @@ class NetEqStreamInput : public test::NetEqInput {
   std::vector<LoggedRtpPacketIncoming>::const_iterator packet_stream_it_;
   std::vector<LoggedAudioPlayoutEvent>::const_iterator output_events_it_;
   const std::vector<LoggedAudioPlayoutEvent>::const_iterator output_events_end_;
+  std::vector<LoggedNetEqSetMinimumDelayEvent>::const_iterator
+      neteq_set_minimum_delay_events_it_;
+  const std::vector<LoggedNetEqSetMinimumDelayEvent>::const_iterator
+      neteq_set_minimum_delay_events_end_;
   const absl::optional<int64_t> end_time_ms_;
 };
 
@@ -294,11 +325,14 @@ class ReplacementAudioDecoderFactory : public AudioDecoderFactory {
 std::unique_ptr<test::NetEqStatsGetter> CreateNetEqTestAndRun(
     const std::vector<LoggedRtpPacketIncoming>* packet_stream,
     const std::vector<LoggedAudioPlayoutEvent>* output_events,
+    const std::vector<LoggedNetEqSetMinimumDelayEvent>*
+        neteq_set_minimum_delay_events,
     absl::optional<int64_t> end_time_ms,
     const std::string& replacement_file_name,
     int file_sample_rate_hz) {
   std::unique_ptr<test::NetEqInput> input(
-      new NetEqStreamInput(packet_stream, output_events, end_time_ms));
+      new NetEqStreamInput(packet_stream, output_events,
+                           neteq_set_minimum_delay_events, end_time_ms));
 
   constexpr int kReplacementPt = 127;
   std::set<uint8_t> cn_types;
@@ -361,11 +395,23 @@ NetEqStatsGetterMap SimulateNetEq(const ParsedRtcEventLog& parsed_log,
       output_events_it = parsed_log.audio_playout_events().cbegin();
     }
 
+    const auto neteq_set_minimum_delay_events_it =
+        parsed_log.neteq_set_minimum_delay_events().find(ssrc);
+    std::vector<LoggedNetEqSetMinimumDelayEvent>
+        empty_neteq_set_minimum_delay_event;
+    const std::vector<LoggedNetEqSetMinimumDelayEvent>&
+        neteq_set_minimum_delay_events =
+            neteq_set_minimum_delay_events_it ==
+                    parsed_log.neteq_set_minimum_delay_events().cend()
+                ? empty_neteq_set_minimum_delay_event
+                : neteq_set_minimum_delay_events_it->second;
+
     int64_t end_time_ms = parsed_log.first_log_segment().stop_time_ms();
 
-    neteq_stats[ssrc] = CreateNetEqTestAndRun(
+    neteq_stats[ssrc] =
-        audio_packets, &output_events_it->second, end_time_ms,
+        CreateNetEqTestAndRun(audio_packets, &output_events_it->second,
-        replacement_file_name, file_sample_rate_hz);
+                              &neteq_set_minimum_delay_events, end_time_ms,
+                              replacement_file_name, file_sample_rate_hz);
   }
 
   return neteq_stats;

test/fuzzers/neteq_rtp_fuzzer.cc

@@ -77,6 +77,10 @@ class FuzzRtpInput : public NetEqInput {
     return input_->NextOutputEventTime();
   }
 
+  absl::optional<SetMinimumDelayInfo> NextSetMinimumDelayInfo() const override {
+    return input_->NextSetMinimumDelayInfo();
+  }
+
   std::unique_ptr<PacketData> PopPacket() override {
     RTC_DCHECK(packet_);
     std::unique_ptr<PacketData> packet_to_return = std::move(packet_);
@@ -88,6 +92,10 @@ class FuzzRtpInput : public NetEqInput {
 
   void AdvanceOutputEvent() override { return input_->AdvanceOutputEvent(); }
 
+  void AdvanceSetMinimumDelay() override {
+    return input_->AdvanceSetMinimumDelay();
+  }
+
   bool ended() const override { return ended_; }
 
   absl::optional<RTPHeader> NextHeader() const override {

test/fuzzers/neteq_signal_fuzzer.cc

@@ -97,6 +97,10 @@ class FuzzSignalInput : public NetEqInput {
     return next_output_event_ms_;
   }
 
+  absl::optional<SetMinimumDelayInfo> NextSetMinimumDelayInfo() const override {
+    return input_->NextSetMinimumDelayInfo();
+  }
+
   std::unique_ptr<PacketData> PopPacket() override {
     RTC_DCHECK(packet_);
     std::unique_ptr<PacketData> packet_to_return = std::move(packet_);
@@ -123,6 +127,10 @@ class FuzzSignalInput : public NetEqInput {
     next_output_event_ms_ += output_event_period_ms_;
   }
 
+  void AdvanceSetMinimumDelay() override {
+    return input_->AdvanceSetMinimumDelay();
+  }
+
   bool ended() const override { return ended_; }
 
   absl::optional<RTPHeader> NextHeader() const override {