diff --git a/modules/audio_coding/neteq/neteq_decoder_plc_unittest.cc b/modules/audio_coding/neteq/neteq_decoder_plc_unittest.cc index 8ed64ffdec..ec6ade9c11 100644 --- a/modules/audio_coding/neteq/neteq_decoder_plc_unittest.cc +++ b/modules/audio_coding/neteq/neteq_decoder_plc_unittest.cc @@ -119,26 +119,34 @@ class LossyInput : public NetEqInput { burst_length_(burst_length), input_(std::move(input)) {} - absl::optional NextEventTime() const { - return input_->NextEventTime(); + absl::optional NextPacketTime() const override { + return input_->NextPacketTime(); } - std::unique_ptr PopEvent() override { - std::unique_ptr event = input_->PopEvent(); - if (event == nullptr || event->type() != Event::Type::kPacketData) { - return event; - } - if (loss_cadence_ != 0 && remaining_losses_ == 0 && - (++count_ % loss_cadence_) == 0) { + absl::optional NextOutputEventTime() const override { + return input_->NextOutputEventTime(); + } + + absl::optional NextSetMinimumDelayInfo() const override { + return input_->NextSetMinimumDelayInfo(); + } + + std::unique_ptr PopPacket() override { + if (loss_cadence_ != 0 && (++count_ % loss_cadence_) == 0) { // Pop `burst_length_` packets to create the loss. - remaining_losses_ = burst_length_; - } else { - if (remaining_losses_ != 0) { - remaining_losses_--; - return PopEvent(); + auto packet_to_return = input_->PopPacket(); + for (int i = 0; i < burst_length_; i++) { + input_->PopPacket(); } + return packet_to_return; } - return event; + return input_->PopPacket(); + } + + void AdvanceOutputEvent() override { return input_->AdvanceOutputEvent(); } + + void AdvanceSetMinimumDelay() override { + return input_->AdvanceSetMinimumDelay(); } bool ended() const override { return input_->ended(); } @@ -150,7 +158,6 @@ class LossyInput : public NetEqInput { private: const int loss_cadence_; const int burst_length_; - int remaining_losses_ = 0; int count_ = 0; const std::unique_ptr input_; }; diff --git a/modules/audio_coding/neteq/neteq_unittest.cc b/modules/audio_coding/neteq/neteq_unittest.cc index 4d7b0b9cf0..fff14bf830 100644 --- a/modules/audio_coding/neteq/neteq_unittest.cc +++ b/modules/audio_coding/neteq/neteq_unittest.cc @@ -988,9 +988,9 @@ TEST(NetEqNoTimeStretchingMode, RunTest) { {5, kRtpExtensionTransportSequenceNumber}, {7, kRtpExtensionVideoContentType}, {8, kRtpExtensionVideoTiming}}; - std::unique_ptr input = std::make_unique( + std::unique_ptr input(new NetEqRtpDumpInput( webrtc::test::ResourcePath("audio_coding/neteq_universal_new", "rtp"), - rtp_ext_map, absl::nullopt /*No SSRC filter*/); + rtp_ext_map, absl::nullopt /*No SSRC filter*/)); std::unique_ptr input_time_limit( new TimeLimitedNetEqInput(std::move(input), 20000)); std::unique_ptr output(new VoidAudioSink); diff --git a/modules/audio_coding/neteq/tools/encode_neteq_input.cc b/modules/audio_coding/neteq/tools/encode_neteq_input.cc index f09aa5b006..87b987ddb6 100644 --- a/modules/audio_coding/neteq/tools/encode_neteq_input.cc +++ b/modules/audio_coding/neteq/tools/encode_neteq_input.cc @@ -24,19 +24,36 @@ EncodeNetEqInput::EncodeNetEqInput(std::unique_ptr generator, : generator_(std::move(generator)), encoder_(std::move(encoder)), input_duration_ms_(input_duration_ms) { - event_ = GetNextEvent(); + CreatePacket(); } EncodeNetEqInput::~EncodeNetEqInput() = default; -std::unique_ptr EncodeNetEqInput::PopEvent() { - std::unique_ptr event_to_return = std::move(event_); - event_ = GetNextEvent(); - return event_to_return; +absl::optional EncodeNetEqInput::NextPacketTime() const { + RTC_DCHECK(packet_data_); + return static_cast(packet_data_->time_ms); +} + +absl::optional EncodeNetEqInput::NextOutputEventTime() const { + return next_output_event_ms_; +} + +std::unique_ptr EncodeNetEqInput::PopPacket() { + RTC_DCHECK(packet_data_); + // Grab the packet to return... + std::unique_ptr packet_to_return = std::move(packet_data_); + // ... and line up the next packet for future use. + CreatePacket(); + + return packet_to_return; +} + +void EncodeNetEqInput::AdvanceOutputEvent() { + next_output_event_ms_ += kOutputPeriodMs; } bool EncodeNetEqInput::ended() const { - return next_output_event_ms_ > input_duration_ms_ + kOutputPeriodMs; + return next_output_event_ms_ > input_duration_ms_; } absl::optional EncodeNetEqInput::NextHeader() const { @@ -44,23 +61,9 @@ absl::optional EncodeNetEqInput::NextHeader() const { return packet_data_->header; } -std::unique_ptr EncodeNetEqInput::GetNextEvent() { - std::unique_ptr event; - if (packet_data_ == nullptr) { - CreatePacket(); - } - if (next_output_event_ms_ < packet_data_->timestamp_ms_) { - event = std::make_unique(next_output_event_ms_); - next_output_event_ms_ += kOutputPeriodMs; - return event; - } - event = std::move(packet_data_); - CreatePacket(); - return event; -} - void EncodeNetEqInput::CreatePacket() { // Create a new PacketData object. + RTC_DCHECK(!packet_data_); packet_data_.reset(new NetEqInput::PacketData); RTC_DCHECK_EQ(packet_data_->payload.size(), 0); @@ -83,7 +86,7 @@ void EncodeNetEqInput::CreatePacket() { packet_data_->header.timestamp = info.encoded_timestamp; packet_data_->header.payloadType = info.payload_type; packet_data_->header.sequenceNumber = sequence_number_++; - packet_data_->timestamp_ms_ = next_packet_time_ms_; + packet_data_->time_ms = next_packet_time_ms_; next_packet_time_ms_ += num_blocks * kOutputPeriodMs; } diff --git a/modules/audio_coding/neteq/tools/encode_neteq_input.h b/modules/audio_coding/neteq/tools/encode_neteq_input.h index f028e4f20a..f2ed4b1cf5 100644 --- a/modules/audio_coding/neteq/tools/encode_neteq_input.h +++ b/modules/audio_coding/neteq/tools/encode_neteq_input.h @@ -37,24 +37,28 @@ class EncodeNetEqInput : public NetEqInput { int64_t input_duration_ms); ~EncodeNetEqInput() override; - std::unique_ptr PopEvent() override; + absl::optional NextPacketTime() const override; + + absl::optional NextOutputEventTime() const override; + + absl::optional NextSetMinimumDelayInfo() const override { + return absl::nullopt; + } + + std::unique_ptr PopPacket() override; + + void AdvanceOutputEvent() override; + + void AdvanceSetMinimumDelay() override {} bool ended() const override; absl::optional NextHeader() const override; - absl::optional NextEventTime() const override { - if (event_) { - return event_->timestamp_ms(); - } - return absl::nullopt; - } - private: static constexpr int64_t kOutputPeriodMs = 10; void CreatePacket(); - std::unique_ptr GetNextEvent(); std::unique_ptr generator_; std::unique_ptr encoder_; @@ -64,7 +68,6 @@ class EncodeNetEqInput : public NetEqInput { int64_t next_packet_time_ms_ = 0; int64_t next_output_event_ms_ = 0; const int64_t input_duration_ms_; - std::unique_ptr event_; }; } // namespace test diff --git a/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.cc b/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.cc index cd5955b9f7..763078eed2 100644 --- a/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.cc +++ b/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.cc @@ -27,27 +27,37 @@ InitialPacketInserterNetEqInput::InitialPacketInserterNetEqInput( packets_to_insert_(number_of_initial_packets), sample_rate_hz_(sample_rate_hz) {} -absl::optional InitialPacketInserterNetEqInput::NextEventTime() const { - return source_->NextEventTime(); +absl::optional InitialPacketInserterNetEqInput::NextPacketTime() + const { + return source_->NextPacketTime(); } -std::unique_ptr InitialPacketInserterNetEqInput::PopEvent() { - std::unique_ptr event; +absl::optional InitialPacketInserterNetEqInput::NextOutputEventTime() + const { + return source_->NextOutputEventTime(); +} + +absl::optional +InitialPacketInserterNetEqInput::NextSetMinimumDelayInfo() const { + return source_->NextSetMinimumDelayInfo(); +} + +std::unique_ptr +InitialPacketInserterNetEqInput::PopPacket() { if (!first_packet_) { - event = source_->PopEvent(); - if (event == nullptr || event->type() != Event::Type::kPacketData) { - return event; + first_packet_ = source_->PopPacket(); + if (!first_packet_) { + // The source has no packets, so we should not insert any dummy packets. + packets_to_insert_ = 0; } - first_packet_ = std::move(event); } if (packets_to_insert_ > 0) { RTC_CHECK(first_packet_); - std::unique_ptr dummy_packet = std::make_unique(); - PacketData& first_packet = static_cast(*first_packet_); - dummy_packet->header = first_packet.header; - dummy_packet->payload = - rtc::Buffer(first_packet.payload.data(), first_packet.payload.size()); - dummy_packet->timestamp_ms_ = first_packet.timestamp_ms_; + auto dummy_packet = std::unique_ptr(new PacketData()); + dummy_packet->header = first_packet_->header; + dummy_packet->payload = rtc::Buffer(first_packet_->payload.data(), + first_packet_->payload.size()); + dummy_packet->time_ms = first_packet_->time_ms; dummy_packet->header.sequenceNumber -= packets_to_insert_; // This assumes 20ms per packet. dummy_packet->header.timestamp -= @@ -55,7 +65,15 @@ std::unique_ptr InitialPacketInserterNetEqInput::PopEvent() { packets_to_insert_--; return dummy_packet; } - return source_->PopEvent(); + return source_->PopPacket(); +} + +void InitialPacketInserterNetEqInput::AdvanceSetMinimumDelay() { + source_->AdvanceSetMinimumDelay(); +} + +void InitialPacketInserterNetEqInput::AdvanceOutputEvent() { + source_->AdvanceOutputEvent(); } bool InitialPacketInserterNetEqInput::ended() const { diff --git a/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.h b/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.h index f45c6950d8..7246949956 100644 --- a/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.h +++ b/modules/audio_coding/neteq/tools/initial_packet_inserter_neteq_input.h @@ -27,16 +27,20 @@ class InitialPacketInserterNetEqInput final : public NetEqInput { InitialPacketInserterNetEqInput(std::unique_ptr source, int number_of_initial_packets, int sample_rate_hz); - std::unique_ptr PopEvent() override; + absl::optional NextPacketTime() const override; + absl::optional NextOutputEventTime() const override; + absl::optional NextSetMinimumDelayInfo() const override; + std::unique_ptr PopPacket() override; + void AdvanceOutputEvent() override; + void AdvanceSetMinimumDelay() override; bool ended() const override; absl::optional NextHeader() const override; - absl::optional NextEventTime() const override; private: const std::unique_ptr source_; int packets_to_insert_; const int sample_rate_hz_; - std::unique_ptr first_packet_; + std::unique_ptr first_packet_; }; } // namespace test diff --git a/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc b/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc index 5e2cf5d95e..9e77457775 100644 --- a/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc +++ b/modules/audio_coding/neteq/tools/neteq_delay_analyzer.cc @@ -87,8 +87,8 @@ void PrintDelays(const NetEqDelayAnalyzer::Delays& delays, void NetEqDelayAnalyzer::AfterInsertPacket( const test::NetEqInput::PacketData& packet, NetEq* neteq) { - data_.insert(std::make_pair(packet.header.timestamp, - TimingData(packet.timestamp_ms()))); + data_.insert( + std::make_pair(packet.header.timestamp, TimingData(packet.time_ms))); ssrcs_.insert(packet.header.ssrc); payload_types_.insert(packet.header.payloadType); } diff --git a/modules/audio_coding/neteq/tools/neteq_event_log_input.cc b/modules/audio_coding/neteq/tools/neteq_event_log_input.cc index c187ba61c4..26255b7771 100644 --- a/modules/audio_coding/neteq/tools/neteq_event_log_input.cc +++ b/modules/audio_coding/neteq/tools/neteq_event_log_input.cc @@ -42,88 +42,36 @@ NetEqEventLogInput* NetEqEventLogInput::CreateFromString( return new NetEqEventLogInput(std::move(event_log_src)); } -std::unique_ptr NetEqEventLogInput::PopEvent() { - std::unique_ptr event_to_return = std::move(event_); - event_ = GetNextEvent(); - return event_to_return; +absl::optional NetEqEventLogInput::NextOutputEventTime() const { + return next_output_event_ms_; } -absl::optional NetEqEventLogInput::NextHeader() const { - return packet_ ? absl::optional(packet_->header()) : absl::nullopt; +absl::optional +NetEqEventLogInput::NextSetMinimumDelayInfo() const { + return next_minimum_delay_event_info_; } -absl::optional NetEqEventLogInput::NextOutputEventTime() { - absl::optional next_output_event_ms; - next_output_event_ms = source_->NextAudioOutputEventMs(); - if (*next_output_event_ms == std::numeric_limits::max()) { - next_output_event_ms = absl::nullopt; +void NetEqEventLogInput::AdvanceOutputEvent() { + next_output_event_ms_ = source_->NextAudioOutputEventMs(); + if (*next_output_event_ms_ == std::numeric_limits::max()) { + next_output_event_ms_ = absl::nullopt; } - return next_output_event_ms; } -std::unique_ptr NetEqEventLogInput::CreatePacketEvent() { - std::unique_ptr packet_data = std::make_unique(); - packet_data->header = packet_->header(); - if (packet_->payload_length_bytes() == 0 && - packet_->virtual_payload_length_bytes() > 0) { - // This is a header-only "dummy" packet. Set the payload to all zeros, with - // length according to the virtual length. - packet_data->payload.SetSize(packet_->virtual_payload_length_bytes()); - std::fill_n(packet_data->payload.data(), packet_data->payload.size(), 0); - } else { - packet_data->payload.SetData(packet_->payload(), - packet_->payload_length_bytes()); - } - packet_data->timestamp_ms_ = packet_->time_ms(); - packet_ = source_->NextPacket(); - return packet_data; +void NetEqEventLogInput::AdvanceSetMinimumDelay() { + next_minimum_delay_event_info_ = source_->NextSetMinimumDelayEvent(); } -std::unique_ptr NetEqEventLogInput::CreateOutputEvent() { - std::unique_ptr event = - std::make_unique(next_output_event_ms_.value()); - next_output_event_ms_ = NextOutputEventTime(); - return event; -} - -std::unique_ptr -NetEqEventLogInput::CreateSetMinimumDelayEvent() { - std::unique_ptr event = - std::make_unique( - next_minimum_delay_event_.value()); - next_minimum_delay_event_ = source_->NextSetMinimumDelayEvent(); - return event; -} - -std::unique_ptr NetEqEventLogInput::GetNextEvent() { - std::unique_ptr event; - int64_t packet_time_ms = packet_ ? static_cast(packet_->time_ms()) - : std::numeric_limits::max(); - int64_t output_time_ms = next_output_event_ms_.has_value() - ? next_output_event_ms_.value() - : std::numeric_limits::max(); - int64_t minimum_delay_ms = next_minimum_delay_event_.has_value() - ? next_minimum_delay_event_->timestamp_ms() - : std::numeric_limits::max(); - if (packet_ && packet_time_ms <= minimum_delay_ms && - packet_time_ms <= output_time_ms) { - event = CreatePacketEvent(); - } else if (next_minimum_delay_event_.has_value() && - minimum_delay_ms <= output_time_ms) { - event = CreateSetMinimumDelayEvent(); - } else if (next_output_event_ms_.has_value()) { - event = CreateOutputEvent(); - } - return event; +PacketSource* NetEqEventLogInput::source() { + return source_.get(); } NetEqEventLogInput::NetEqEventLogInput( std::unique_ptr source) : source_(std::move(source)) { - packet_ = source_->NextPacket(); - next_minimum_delay_event_ = source_->NextSetMinimumDelayEvent(); - next_output_event_ms_ = NextOutputEventTime(); - event_ = GetNextEvent(); + LoadNextPacket(); + AdvanceOutputEvent(); + AdvanceSetMinimumDelay(); } } // namespace test diff --git a/modules/audio_coding/neteq/tools/neteq_event_log_input.h b/modules/audio_coding/neteq/tools/neteq_event_log_input.h index 0ab09f4811..1e64cc59a4 100644 --- a/modules/audio_coding/neteq/tools/neteq_event_log_input.h +++ b/modules/audio_coding/neteq/tools/neteq_event_log_input.h @@ -24,8 +24,9 @@ namespace test { class RtcEventLogSource; -// Implementation of a NetEqInput from an RtcEventLogSource. -class NetEqEventLogInput final : public NetEqInput { +// Implementation of NetEqPacketSourceInput to be used with an +// RtcEventLogSource. +class NetEqEventLogInput final : public NetEqPacketSourceInput { public: static NetEqEventLogInput* CreateFromFile( absl::string_view file_name, @@ -34,31 +35,18 @@ class NetEqEventLogInput final : public NetEqInput { absl::string_view file_contents, absl::optional ssrc_filter); - absl::optional NextEventTime() const override { - if (event_) { - return event_->timestamp_ms(); - } - return absl::nullopt; - } - std::unique_ptr PopEvent() override; - absl::optional NextHeader() const override; - bool ended() const override { - return !next_output_event_ms_ || packet_ == nullptr; - } + absl::optional NextOutputEventTime() const override; + absl::optional NextSetMinimumDelayInfo() const override; + void AdvanceOutputEvent() override; + void AdvanceSetMinimumDelay() override; + + protected: + PacketSource* source() override; private: NetEqEventLogInput(std::unique_ptr source); - std::unique_ptr GetNextEvent(); - std::unique_ptr CreatePacketEvent(); - std::unique_ptr CreateOutputEvent(); - std::unique_ptr CreateSetMinimumDelayEvent(); - absl::optional NextOutputEventTime(); - std::unique_ptr source_; - std::unique_ptr packet_; - absl::optional next_output_event_ms_; - absl::optional next_minimum_delay_event_; - std::unique_ptr event_; + absl::optional next_minimum_delay_event_info_; }; } // namespace test diff --git a/modules/audio_coding/neteq/tools/neteq_input.cc b/modules/audio_coding/neteq/tools/neteq_input.cc index d47a7d0ab7..57160b9969 100644 --- a/modules/audio_coding/neteq/tools/neteq_input.cc +++ b/modules/audio_coding/neteq/tools/neteq_input.cc @@ -22,7 +22,7 @@ std::string NetEqInput::PacketData::ToString() const { rtc::StringBuilder ss; ss << "{" "time_ms: " - << static_cast(timestamp_ms_) + << static_cast(time_ms) << ", " "header: {" "pt: " @@ -50,24 +50,46 @@ TimeLimitedNetEqInput::TimeLimitedNetEqInput(std::unique_ptr input, TimeLimitedNetEqInput::~TimeLimitedNetEqInput() = default; -std::unique_ptr TimeLimitedNetEqInput::PopEvent() { - std::unique_ptr event; +absl::optional TimeLimitedNetEqInput::NextPacketTime() const { + return ended_ ? absl::nullopt : input_->NextPacketTime(); +} + +absl::optional TimeLimitedNetEqInput::NextOutputEventTime() const { + return ended_ ? absl::nullopt : input_->NextOutputEventTime(); +} + +absl::optional +TimeLimitedNetEqInput::NextSetMinimumDelayInfo() const { + return ended_ ? absl::nullopt : input_->NextSetMinimumDelayInfo(); +} + +std::unique_ptr TimeLimitedNetEqInput::PopPacket() { if (ended_) { - return event; + return std::unique_ptr(); } - event = input_->PopEvent(); + auto packet = input_->PopPacket(); MaybeSetEnded(); - return event; + return packet; +} + +void TimeLimitedNetEqInput::AdvanceOutputEvent() { + if (!ended_) { + input_->AdvanceOutputEvent(); + MaybeSetEnded(); + } +} + +void TimeLimitedNetEqInput::AdvanceSetMinimumDelay() { + if (!ended_) { + input_->AdvanceSetMinimumDelay(); + MaybeSetEnded(); + } } bool TimeLimitedNetEqInput::ended() const { return ended_ || input_->ended(); } -absl::optional TimeLimitedNetEqInput::NextEventTime() const { - return input_->NextEventTime(); -} - absl::optional TimeLimitedNetEqInput::NextHeader() const { return ended_ ? absl::nullopt : input_->NextHeader(); } diff --git a/modules/audio_coding/neteq/tools/neteq_input.h b/modules/audio_coding/neteq/tools/neteq_input.h index 8c6b1c33f3..56b0212add 100644 --- a/modules/audio_coding/neteq/tools/neteq_input.h +++ b/modules/audio_coding/neteq/tools/neteq_input.h @@ -26,53 +26,74 @@ namespace test { // Interface class for input to the NetEqTest class. class NetEqInput { public: - class Event { - public: - enum class Type { kPacketData, kGetAudio, kSetMinimumDelay }; - virtual Type type() = 0; - virtual int64_t timestamp_ms() const = 0; - virtual ~Event() = default; - }; - - class PacketData : public Event { - public: + struct PacketData { PacketData(); ~PacketData(); - Type type() override { return Type::kPacketData; } - int64_t timestamp_ms() const override { return timestamp_ms_; } std::string ToString() const; + RTPHeader header; rtc::Buffer payload; - int64_t timestamp_ms_; + int64_t time_ms; }; - class SetMinimumDelay : public Event { - public: - SetMinimumDelay(int64_t timestamp_ms_in, int delay_ms_in) - : timestamp_ms_(timestamp_ms_in), delay_ms_(delay_ms_in) {} - Type type() override { return Type::kSetMinimumDelay; } - int64_t timestamp_ms() const override { return timestamp_ms_; } - int delay_ms() { return delay_ms_; } - - private: - int64_t timestamp_ms_; - int delay_ms_; - }; - - class GetAudio : public Event { - public: - explicit GetAudio(int64_t timestamp_ms_in) - : timestamp_ms_(timestamp_ms_in) {} - Type type() override { return Type::kGetAudio; } - int64_t timestamp_ms() const override { return timestamp_ms_; } - - private: - int64_t timestamp_ms_; + struct SetMinimumDelayInfo { + SetMinimumDelayInfo(int64_t timestamp_ms_in, int delay_ms_in) + : timestamp_ms(timestamp_ms_in), delay_ms(delay_ms_in) {} + int64_t timestamp_ms; + int delay_ms; }; virtual ~NetEqInput() = default; - virtual std::unique_ptr PopEvent() = 0; + // Returns at what time (in ms) NetEq::InsertPacket should be called next, or + // empty if the source is out of packets. + virtual absl::optional NextPacketTime() const = 0; + + // Returns at what time (in ms) NetEq::GetAudio should be called next, or + // empty if no more output events are available. + virtual absl::optional NextOutputEventTime() const = 0; + + // Returns the information related to the next NetEq set minimum delay event + // if available. + virtual absl::optional NextSetMinimumDelayInfo() + const = 0; + + // Returns the time (in ms) for the next event (packet, output or set minimum + // delay event) or empty if there are no more events. + absl::optional NextEventTime() const { + absl::optional next_event_time = NextPacketTime(); + const auto next_output_time = NextOutputEventTime(); + // Return the minimum of non-empty `a` and `b`, or empty if both are empty. + if (next_output_time) { + next_event_time = next_event_time ? std::min(next_event_time.value(), + next_output_time.value()) + : next_output_time; + } + const auto next_neteq_minimum_delay = NextSetMinimumDelayInfo(); + if (next_neteq_minimum_delay) { + next_event_time = + next_event_time + ? std::min(next_event_time.value(), + next_neteq_minimum_delay.value().timestamp_ms) + : next_neteq_minimum_delay.value().timestamp_ms; + } + return next_event_time; + } + + // Returns the next packet to be inserted into NetEq. The packet following the + // returned one is pre-fetched in the NetEqInput object, such that future + // calls to NextPacketTime() or NextHeader() will return information from that + // packet. + virtual std::unique_ptr PopPacket() = 0; + + // Move to the next output event. This will make NextOutputEventTime() return + // a new value (potentially the same if several output events share the same + // time). + virtual void AdvanceOutputEvent() = 0; + + // Move to the next NetEq set minimum delay. This will make + // `NextSetMinimumDelayInfo` return a new value. + virtual void AdvanceSetMinimumDelay() = 0; // Returns true if the source has come to an end. An implementation must // eventually return true from this method, or the test will end up in an @@ -82,10 +103,6 @@ class NetEqInput { // Returns the RTP header for the next packet, i.e., the packet that will be // delivered next by PopPacket(). virtual absl::optional NextHeader() const = 0; - - // Returns the time (in ms) for the next event, or empty if both are out of - // events. - virtual absl::optional NextEventTime() const = 0; }; // Wrapper class to impose a time limit on a NetEqInput object, typically @@ -95,8 +112,12 @@ class TimeLimitedNetEqInput : public NetEqInput { public: TimeLimitedNetEqInput(std::unique_ptr input, int64_t duration_ms); ~TimeLimitedNetEqInput() override; - absl::optional NextEventTime() const override; - std::unique_ptr PopEvent() override; + absl::optional NextPacketTime() const override; + absl::optional NextOutputEventTime() const override; + absl::optional NextSetMinimumDelayInfo() const override; + std::unique_ptr PopPacket() override; + void AdvanceOutputEvent() override; + void AdvanceSetMinimumDelay() override; bool ended() const override; absl::optional NextHeader() const override; diff --git a/modules/audio_coding/neteq/tools/neteq_packet_source_input.cc b/modules/audio_coding/neteq/tools/neteq_packet_source_input.cc index f5b5e73152..55a5653238 100644 --- a/modules/audio_coding/neteq/tools/neteq_packet_source_input.cc +++ b/modules/audio_coding/neteq/tools/neteq_packet_source_input.cc @@ -20,45 +20,27 @@ namespace webrtc { namespace test { -NetEqPacketSourceInput::NetEqPacketSourceInput( - absl::string_view file_name, - const NetEqPacketSourceInput::RtpHeaderExtensionMap& hdr_ext_map, - absl::optional ssrc_filter) - : next_output_event_ms_(0) { - std::unique_ptr source( - RtpFileSource::Create(file_name, ssrc_filter)); - for (const auto& ext_pair : hdr_ext_map) { - source->RegisterRtpHeaderExtension(ext_pair.second, ext_pair.first); - } - packet_source_ = std::move(source); - packet_ = packet_source_->NextPacket(); - event_ = GetNextEvent(); -} +NetEqPacketSourceInput::NetEqPacketSourceInput() : next_output_event_ms_(0) {} -std::unique_ptr NetEqPacketSourceInput::PopEvent() { - std::unique_ptr event_to_return = std::move(event_); - event_ = GetNextEvent(); - return event_to_return; +absl::optional NetEqPacketSourceInput::NextPacketTime() const { + return packet_ + ? absl::optional(static_cast(packet_->time_ms())) + : absl::nullopt; } absl::optional NetEqPacketSourceInput::NextHeader() const { - if (packet_) { - return packet_->header(); - } - return absl::nullopt; + return packet_ ? absl::optional(packet_->header()) : absl::nullopt; } -std::unique_ptr NetEqPacketSourceInput::GetNextEvent() { +void NetEqPacketSourceInput::LoadNextPacket() { + packet_ = source()->NextPacket(); +} + +std::unique_ptr NetEqPacketSourceInput::PopPacket() { if (!packet_) { - return nullptr; + return std::unique_ptr(); } - if (packet_->time_ms() > next_output_event_ms_) { - std::unique_ptr event = - std::make_unique(next_output_event_ms_); - next_output_event_ms_ += kOutputPeriodMs; - return event; - } - std::unique_ptr packet_data = std::make_unique(); + std::unique_ptr packet_data(new PacketData); packet_data->header = packet_->header(); if (packet_->payload_length_bytes() == 0 && packet_->virtual_payload_length_bytes() > 0) { @@ -70,10 +52,39 @@ std::unique_ptr NetEqPacketSourceInput::GetNextEvent() { packet_data->payload.SetData(packet_->payload(), packet_->payload_length_bytes()); } - packet_data->timestamp_ms_ = packet_->time_ms(); - packet_ = packet_source_->NextPacket(); + packet_data->time_ms = packet_->time_ms(); + + LoadNextPacket(); + return packet_data; } +NetEqRtpDumpInput::NetEqRtpDumpInput(absl::string_view file_name, + const RtpHeaderExtensionMap& hdr_ext_map, + absl::optional ssrc_filter) + : source_(RtpFileSource::Create(file_name, ssrc_filter)) { + for (const auto& ext_pair : hdr_ext_map) { + source_->RegisterRtpHeaderExtension(ext_pair.second, ext_pair.first); + } + LoadNextPacket(); +} + +absl::optional NetEqRtpDumpInput::NextOutputEventTime() const { + return next_output_event_ms_; +} + +void NetEqRtpDumpInput::AdvanceOutputEvent() { + if (next_output_event_ms_) { + *next_output_event_ms_ += kOutputPeriodMs; + } + if (!NextPacketTime()) { + next_output_event_ms_ = absl::nullopt; + } +} + +PacketSource* NetEqRtpDumpInput::source() { + return source_.get(); +} + } // namespace test } // namespace webrtc diff --git a/modules/audio_coding/neteq/tools/neteq_packet_source_input.h b/modules/audio_coding/neteq/tools/neteq_packet_source_input.h index 6c78850f09..1b48b1f8cf 100644 --- a/modules/audio_coding/neteq/tools/neteq_packet_source_input.h +++ b/modules/audio_coding/neteq/tools/neteq_packet_source_input.h @@ -30,29 +30,43 @@ class NetEqPacketSourceInput : public NetEqInput { public: using RtpHeaderExtensionMap = std::map; - NetEqPacketSourceInput( - absl::string_view file_name, - const NetEqPacketSourceInput::RtpHeaderExtensionMap& hdr_ext_map, - absl::optional ssrc_filter); + NetEqPacketSourceInput(); + absl::optional NextPacketTime() const override; + std::unique_ptr PopPacket() override; + absl::optional NextHeader() const override; + bool ended() const override { return !next_output_event_ms_; } - absl::optional NextEventTime() const override { - if (event_) { - return event_->timestamp_ms(); - } + protected: + virtual PacketSource* source() = 0; + void LoadNextPacket(); + + absl::optional next_output_event_ms_; + + private: + std::unique_ptr packet_; +}; + +// Implementation of NetEqPacketSourceInput to be used with an RtpFileSource. +class NetEqRtpDumpInput final : public NetEqPacketSourceInput { + public: + NetEqRtpDumpInput(absl::string_view file_name, + const RtpHeaderExtensionMap& hdr_ext_map, + absl::optional ssrc_filter); + + absl::optional NextOutputEventTime() const override; + absl::optional NextSetMinimumDelayInfo() const override { return absl::nullopt; } - std::unique_ptr PopEvent() override; - absl::optional NextHeader() const override; - bool ended() const override { return event_ == nullptr; } + void AdvanceOutputEvent() override; + void AdvanceSetMinimumDelay() override {} + + protected: + PacketSource* source() override; private: static constexpr int64_t kOutputPeriodMs = 10; - std::unique_ptr GetNextEvent(); - std::unique_ptr packet_; - std::unique_ptr packet_source_; - int64_t next_output_event_ms_; - std::unique_ptr event_; + std::unique_ptr source_; }; } // namespace test diff --git a/modules/audio_coding/neteq/tools/neteq_replacement_input.cc b/modules/audio_coding/neteq/tools/neteq_replacement_input.cc index 1a2cff2a3b..9436b68ac9 100644 --- a/modules/audio_coding/neteq/tools/neteq_replacement_input.cc +++ b/modules/audio_coding/neteq/tools/neteq_replacement_input.cc @@ -26,25 +26,46 @@ NetEqReplacementInput::NetEqReplacementInput( comfort_noise_types_(comfort_noise_types), forbidden_types_(forbidden_types) { RTC_CHECK(source_); - event_ = source_->PopEvent(); - ReplaceIfPacketEvent(); + packet_ = source_->PopPacket(); + ReplacePacket(); } -std::unique_ptr NetEqReplacementInput::PopEvent() { - std::unique_ptr event_to_return = std::move(event_); +absl::optional NetEqReplacementInput::NextPacketTime() const { + return packet_ + ? absl::optional(static_cast(packet_->time_ms)) + : absl::nullopt; +} + +absl::optional NetEqReplacementInput::NextOutputEventTime() const { + return source_->NextOutputEventTime(); +} + +absl::optional +NetEqReplacementInput::NextSetMinimumDelayInfo() const { + return source_->NextSetMinimumDelayInfo(); +} + +std::unique_ptr NetEqReplacementInput::PopPacket() { + std::unique_ptr to_return = std::move(packet_); while (true) { - event_ = source_->PopEvent(); - if (event_ == nullptr || event_->type() != Event::Type::kPacketData) { + packet_ = source_->PopPacket(); + if (!packet_) break; - } - PacketData& packet = static_cast(*event_); - if (packet.payload.size() > packet.header.paddingLength) { + if (packet_->payload.size() > packet_->header.paddingLength) { // Not padding only. Good to go. Skip this packet otherwise. break; } } - ReplaceIfPacketEvent(); - return event_to_return; + ReplacePacket(); + return to_return; +} + +void NetEqReplacementInput::AdvanceOutputEvent() { + source_->AdvanceOutputEvent(); +} + +void NetEqReplacementInput::AdvanceSetMinimumDelay() { + source_->AdvanceSetMinimumDelay(); } bool NetEqReplacementInput::ended() const { @@ -55,38 +76,36 @@ absl::optional NetEqReplacementInput::NextHeader() const { return source_->NextHeader(); } -void NetEqReplacementInput::ReplaceIfPacketEvent() { - if (event_ == nullptr || event_->type() != Event::Type::kPacketData) { +void NetEqReplacementInput::ReplacePacket() { + if (!source_->NextPacketTime()) { + // End of input. Cannot do proper replacement on the very last packet, so we + // delete it instead. + packet_.reset(); return; } - PacketData& packet = static_cast(*event_); + RTC_DCHECK(packet_); - RTC_CHECK_EQ(forbidden_types_.count(packet.header.payloadType), 0) - << "Payload type " << static_cast(packet.header.payloadType) + RTC_CHECK_EQ(forbidden_types_.count(packet_->header.payloadType), 0) + << "Payload type " << static_cast(packet_->header.payloadType) << " is forbidden."; // Check if this packet is comfort noise. - if (comfort_noise_types_.count(packet.header.payloadType) != 0) { + if (comfort_noise_types_.count(packet_->header.payloadType) != 0) { // If CNG, simply insert a zero-energy one-byte payload. uint8_t cng_payload[1] = {127}; // Max attenuation of CNG. - packet.payload.SetData(cng_payload); + packet_->payload.SetData(cng_payload); return; } absl::optional next_hdr = source_->NextHeader(); - if (!next_hdr) { - // End of input. Cannot do proper replacement on the very last packet, so we - // delete it instead. - event_ = nullptr; - return; - } RTC_DCHECK(next_hdr); uint8_t payload[12]; RTC_DCHECK_LE(last_frame_size_timestamps_, 120 * 48); uint32_t input_frame_size_timestamps = last_frame_size_timestamps_; - const uint32_t timestamp_diff = next_hdr->timestamp - packet.header.timestamp; - if (next_hdr->sequenceNumber == packet.header.sequenceNumber + 1 && + const uint32_t timestamp_diff = + next_hdr->timestamp - packet_->header.timestamp; + if (next_hdr->sequenceNumber == packet_->header.sequenceNumber + 1 && timestamp_diff <= 120 * 48) { // Packets are in order and the timestamp diff is less than 5760 samples. // Accept the timestamp diff as a valid frame size. @@ -94,11 +113,11 @@ void NetEqReplacementInput::ReplaceIfPacketEvent() { last_frame_size_timestamps_ = input_frame_size_timestamps; } RTC_DCHECK_LE(input_frame_size_timestamps, 120 * 48); - FakeDecodeFromFile::PrepareEncoded(packet.header.timestamp, + FakeDecodeFromFile::PrepareEncoded(packet_->header.timestamp, input_frame_size_timestamps, - packet.payload.size(), payload); - packet.payload.SetData(payload); - packet.header.payloadType = replacement_payload_type_; + packet_->payload.size(), payload); + packet_->payload.SetData(payload); + packet_->header.payloadType = replacement_payload_type_; return; } diff --git a/modules/audio_coding/neteq/tools/neteq_replacement_input.h b/modules/audio_coding/neteq/tools/neteq_replacement_input.h index 8f76494461..23e4beae84 100644 --- a/modules/audio_coding/neteq/tools/neteq_replacement_input.h +++ b/modules/audio_coding/neteq/tools/neteq_replacement_input.h @@ -28,24 +28,23 @@ class NetEqReplacementInput : public NetEqInput { const std::set& comfort_noise_types, const std::set& forbidden_types); - absl::optional NextEventTime() const override { - if (event_) { - return event_->timestamp_ms(); - } - return absl::nullopt; - } - std::unique_ptr PopEvent() override; + absl::optional NextPacketTime() const override; + absl::optional NextOutputEventTime() const override; + absl::optional NextSetMinimumDelayInfo() const override; + std::unique_ptr PopPacket() override; + void AdvanceOutputEvent() override; + void AdvanceSetMinimumDelay() override; bool ended() const override; absl::optional NextHeader() const override; private: - void ReplaceIfPacketEvent(); + void ReplacePacket(); std::unique_ptr source_; const uint8_t replacement_payload_type_; const std::set comfort_noise_types_; const std::set forbidden_types_; - std::unique_ptr event_; // The next event to deliver. + std::unique_ptr packet_; // The next packet to deliver. uint32_t last_frame_size_timestamps_ = 960; // Initial guess: 20 ms @ 48 kHz. }; diff --git a/modules/audio_coding/neteq/tools/neteq_test.cc b/modules/audio_coding/neteq/tools/neteq_test.cc index 3b3f9f54db..125d087157 100644 --- a/modules/audio_coding/neteq/tools/neteq_test.cc +++ b/modules/audio_coding/neteq/tools/neteq_test.cc @@ -107,27 +107,25 @@ NetEqTest::SimulationStepResult NetEqTest::RunToNextGetAudio() { RTC_DCHECK(input_->NextEventTime()); clock_.AdvanceTimeMilliseconds(*input_->NextEventTime() - time_now_ms); time_now_ms = *input_->NextEventTime(); - std::unique_ptr event = input_->PopEvent(); // Check if it is time to insert packet. - RTC_CHECK(event); - if (event->type() == NetEqInput::Event::Type::kPacketData) { - NetEqInput::PacketData& packet_data = - static_cast(*event); + if (input_->NextPacketTime() && time_now_ms >= *input_->NextPacketTime()) { + std::unique_ptr packet_data = input_->PopPacket(); + RTC_CHECK(packet_data); const size_t payload_data_length = - packet_data.payload.size() - packet_data.header.paddingLength; + packet_data->payload.size() - packet_data->header.paddingLength; if (payload_data_length != 0) { int error = neteq_->InsertPacket( - packet_data.header, - rtc::ArrayView(packet_data.payload)); + packet_data->header, + rtc::ArrayView(packet_data->payload)); if (error != NetEq::kOK && callbacks_.error_callback) { - callbacks_.error_callback->OnInsertPacketError(packet_data); + callbacks_.error_callback->OnInsertPacketError(*packet_data); } if (callbacks_.post_insert_packet) { - callbacks_.post_insert_packet->AfterInsertPacket(packet_data, + callbacks_.post_insert_packet->AfterInsertPacket(*packet_data, neteq_.get()); } } else { - neteq_->InsertEmptyPacket(packet_data.header); + neteq_->InsertEmptyPacket(packet_data->header); } if (last_packet_time_ms_) { current_state_.packet_iat_ms.push_back(time_now_ms - @@ -139,20 +137,20 @@ NetEqTest::SimulationStepResult NetEqTest::RunToNextGetAudio() { last_packet_time_ms_ ? (time_now_ms - *last_packet_time_ms_) : -1; const auto delta_timestamp = last_packet_timestamp_ - ? (static_cast(packet_data.header.timestamp) - + ? (static_cast(packet_data->header.timestamp) - *last_packet_timestamp_) * 1000 / sample_rate_hz_ : -1; const auto packet_size_bytes = - packet_data.payload.size() == 12 + packet_data->payload.size() == 12 ? ByteReader::ReadLittleEndian( - &packet_data.payload[8]) + &packet_data->payload[8]) : -1; *text_log_ << "Packet - wallclock: " << std::setw(5) << time_now_ms << ", delta wc: " << std::setw(4) << delta_wallclock - << ", seq_no: " << packet_data.header.sequenceNumber + << ", seq_no: " << packet_data->header.sequenceNumber << ", timestamp: " << std::setw(10) - << packet_data.header.timestamp + << packet_data->header.timestamp << ", delta ts: " << std::setw(4) << delta_timestamp << ", size: " << std::setw(5) << packet_size_bytes << ", frame size: " << std::setw(3) @@ -162,16 +160,19 @@ NetEqTest::SimulationStepResult NetEqTest::RunToNextGetAudio() { } last_packet_time_ms_ = absl::make_optional(time_now_ms); last_packet_timestamp_ = - absl::make_optional(packet_data.header.timestamp); + absl::make_optional(packet_data->header.timestamp); } - if (event->type() == NetEqInput::Event::Type::kSetMinimumDelay) { + if (input_->NextSetMinimumDelayInfo().has_value() && + time_now_ms >= input_->NextSetMinimumDelayInfo().value().timestamp_ms) { neteq_->SetBaseMinimumDelayMs( - static_cast(*event).delay_ms()); + input_->NextSetMinimumDelayInfo().value().delay_ms); + input_->AdvanceSetMinimumDelay(); } // Check if it is time to get output audio. - if (event->type() == NetEqInput::Event::Type::kGetAudio) { + if (input_->NextOutputEventTime() && + time_now_ms >= *input_->NextOutputEventTime()) { if (callbacks_.get_audio_callback) { callbacks_.get_audio_callback->BeforeGetAudio(neteq_.get()); } @@ -199,6 +200,7 @@ NetEqTest::SimulationStepResult NetEqTest::RunToNextGetAudio() { out_frame.samples_per_channel_ * out_frame.num_channels_)); } + input_->AdvanceOutputEvent(); result.simulation_step_ms = input_->NextEventTime().value_or(time_now_ms) - start_time_ms; const auto operations_state = neteq_->GetOperationsAndState(); @@ -269,8 +271,8 @@ NetEqTest::SimulationStepResult NetEqTest::RunToNextGetAudio() { } } prev_lifetime_stats_ = lifetime_stats; - const bool no_more_events = - !input_->NextEventTime() && !operations_state.next_packet_available; + const bool no_more_packets_to_decode = + !input_->NextPacketTime() && !operations_state.next_packet_available; // End the simulation if the gap is too large. This indicates an issue // with the event log file. const bool simulation_step_too_large = result.simulation_step_ms > 1000; @@ -279,8 +281,9 @@ NetEqTest::SimulationStepResult NetEqTest::RunToNextGetAudio() { // the simulation time, which can be a large distortion. result.simulation_step_ms = 10; } - result.is_simulation_finished = - simulation_step_too_large || no_more_events || input_->ended(); + result.is_simulation_finished = simulation_step_too_large || + no_more_packets_to_decode || + input_->ended(); prev_ops_state_ = operations_state; return result; } diff --git a/modules/audio_coding/neteq/tools/neteq_test_factory.cc b/modules/audio_coding/neteq/tools/neteq_test_factory.cc index faf098ccef..6cd371406c 100644 --- a/modules/audio_coding/neteq/tools/neteq_test_factory.cc +++ b/modules/audio_coding/neteq/tools/neteq_test_factory.cc @@ -136,8 +136,8 @@ std::unique_ptr NetEqTestFactory::InitializeTestFromFile( std::unique_ptr input; if (RtpFileSource::ValidRtpDump(input_file_name) || RtpFileSource::ValidPcap(input_file_name)) { - input = std::make_unique( - input_file_name, rtp_ext_map, config.ssrc_filter); + input.reset(new NetEqRtpDumpInput(input_file_name, rtp_ext_map, + config.ssrc_filter)); } else { input.reset(NetEqEventLogInput::CreateFromFile(input_file_name, config.ssrc_filter)); @@ -169,17 +169,21 @@ std::unique_ptr NetEqTestFactory::InitializeTest( if (config.skip_get_audio_events > 0) { std::cout << "Skipping " << config.skip_get_audio_events << " get_audio events" << std::endl; - if (!input->NextEventTime()) { + if (!input->NextPacketTime() || !input->NextOutputEventTime()) { std::cerr << "No events found" << std::endl; return nullptr; } for (int i = 0; i < config.skip_get_audio_events; i++) { - std::unique_ptr event = input->PopEvent(); - while (event && event->type() != NetEqInput::Event::Type::kGetAudio) { - event = input->PopEvent(); + input->AdvanceOutputEvent(); + if (!input->NextOutputEventTime()) { + std::cerr << "Not enough get_audio events found" << std::endl; + return nullptr; } - if (event == nullptr) { - std::cerr << "Not enough events found" << std::endl; + } + while (*input->NextPacketTime() < *input->NextOutputEventTime()) { + input->PopPacket(); + if (!input->NextPacketTime()) { + std::cerr << "Not enough incoming packets found" << std::endl; return nullptr; } } @@ -208,10 +212,7 @@ std::unique_ptr NetEqTestFactory::InitializeTest( // types and SSRCs. discarded_pt_and_ssrc.emplace(first_rtp_header->payloadType, first_rtp_header->ssrc); - std::unique_ptr event = input->PopEvent(); - while (event && event->type() != NetEqInput::Event::Type::kPacketData) { - event = input->PopEvent(); - } + input->PopPacket(); } if (!discarded_pt_and_ssrc.empty()) { std::cout << "Discarded initial packets with the following payload types " diff --git a/modules/audio_coding/neteq/tools/rtc_event_log_source.cc b/modules/audio_coding/neteq/tools/rtc_event_log_source.cc index 64ae891274..68acfb7f62 100644 --- a/modules/audio_coding/neteq/tools/rtc_event_log_source.cc +++ b/modules/audio_coding/neteq/tools/rtc_event_log_source.cc @@ -95,7 +95,7 @@ int64_t RtcEventLogSource::NextAudioOutputEventMs() { return output_time_ms; } -absl::optional +absl::optional RtcEventLogSource::NextSetMinimumDelayEvent() { if (minimum_delay_index_ >= minimum_delay_.size()) { return absl::nullopt; diff --git a/modules/audio_coding/neteq/tools/rtc_event_log_source.h b/modules/audio_coding/neteq/tools/rtc_event_log_source.h index f8c021c313..1445314247 100644 --- a/modules/audio_coding/neteq/tools/rtc_event_log_source.h +++ b/modules/audio_coding/neteq/tools/rtc_event_log_source.h @@ -55,7 +55,7 @@ class RtcEventLogSource : public PacketSource { int64_t NextAudioOutputEventMs(); // Returns the next NetEq set minimum delay event if available. - absl::optional NextSetMinimumDelayEvent(); + absl::optional NextSetMinimumDelayEvent(); private: RtcEventLogSource(); @@ -67,7 +67,7 @@ class RtcEventLogSource : public PacketSource { size_t rtp_packet_index_ = 0; std::vector audio_outputs_; size_t audio_output_index_ = 0; - std::vector minimum_delay_; + std::vector minimum_delay_; size_t minimum_delay_index_ = 0; }; diff --git a/rtc_tools/rtc_event_log_visualizer/analyze_audio.cc b/rtc_tools/rtc_event_log_visualizer/analyze_audio.cc index 38cb55425e..e2b4ecfe8d 100644 --- a/rtc_tools/rtc_event_log_visualizer/analyze_audio.cc +++ b/rtc_tools/rtc_event_log_visualizer/analyze_audio.cc @@ -196,52 +196,9 @@ class NetEqStreamInput : public test::NetEqInput { end_time_ms_(end_time_ms) { RTC_DCHECK(packet_stream); RTC_DCHECK(output_events); - event_ = GetNextEvent(); } - absl::optional NextEventTime() const override { - if (event_) { - return event_->timestamp_ms(); - } - return absl::nullopt; - } - - std::unique_ptr PopEvent() override { - std::unique_ptr return_event = std::move(event_); - event_ = GetNextEvent(); - return return_event; - } - - std::unique_ptr GetNextEvent() { - absl::optional next_time; - absl::optional next_packet_time = NextPacketTime(); - absl::optional next_output_time = NextOutputEventTime(); - absl::optional next_set_minimum_delay_time = - NextSetMinimumDelayTime(); - for (auto time : - {next_packet_time, next_output_time, next_set_minimum_delay_time}) { - if (time) { - if (next_time) { - next_time = time.value() < next_time.value() ? time : next_time; - } else { - next_time = time; - } - } - } - if (next_packet_time && next_packet_time.value() == next_time.value()) { - return PopPacket(); - } - if (next_output_time && next_output_time.value() == next_time.value()) { - return PopGetAudio(); - } - if (next_set_minimum_delay_time && - next_set_minimum_delay_time.value() == next_time.value()) { - return PopNetEqSetMinimumDelay(); - } - return nullptr; - } - - absl::optional NextPacketTime() const { + absl::optional NextPacketTime() const override { if (packet_stream_it_ == packet_stream_.end()) { return absl::nullopt; } @@ -251,7 +208,7 @@ class NetEqStreamInput : public test::NetEqInput { return packet_stream_it_->rtp.log_time_ms(); } - absl::optional NextOutputEventTime() const { + absl::optional NextOutputEventTime() const override { if (output_events_it_ == output_events_end_) { return absl::nullopt; } @@ -261,7 +218,7 @@ class NetEqStreamInput : public test::NetEqInput { return output_events_it_->log_time_ms(); } - absl::optional NextSetMinimumDelayTime() const { + absl::optional NextSetMinimumDelayInfo() const override { if (neteq_set_minimum_delay_events_it_ == neteq_set_minimum_delay_events_end_) { return absl::nullopt; @@ -270,16 +227,18 @@ class NetEqStreamInput : public test::NetEqInput { neteq_set_minimum_delay_events_it_->log_time_ms() > *end_time_ms_) { return absl::nullopt; } - return neteq_set_minimum_delay_events_it_->log_time_ms(); + return SetMinimumDelayInfo( + neteq_set_minimum_delay_events_it_->log_time_ms(), + neteq_set_minimum_delay_events_it_->minimum_delay_ms); } - std::unique_ptr PopPacket() { + std::unique_ptr PopPacket() override { if (packet_stream_it_ == packet_stream_.end()) { return std::unique_ptr(); } std::unique_ptr packet_data(new PacketData()); packet_data->header = packet_stream_it_->rtp.header; - packet_data->timestamp_ms_ = packet_stream_it_->rtp.log_time_ms(); + packet_data->time_ms = packet_stream_it_->rtp.log_time_ms(); // This is a header-only "dummy" packet. Set the payload to all zeros, with // length according to the virtual length. @@ -291,26 +250,17 @@ class NetEqStreamInput : public test::NetEqInput { return packet_data; } - std::unique_ptr PopGetAudio() { - std::unique_ptr get_audio; + void AdvanceOutputEvent() override { if (output_events_it_ != output_events_end_) { - get_audio = std::make_unique(output_events_it_->log_time_ms()); ++output_events_it_; } - return get_audio; } - std::unique_ptr PopNetEqSetMinimumDelay() { - std::unique_ptr net_eq_set_minimum_delay; + void AdvanceSetMinimumDelay() override { if (neteq_set_minimum_delay_events_it_ != neteq_set_minimum_delay_events_end_) { - net_eq_set_minimum_delay = std::make_unique( - neteq_set_minimum_delay_events_it_->log_time_ms(), - neteq_set_minimum_delay_events_it_->minimum_delay_ms); - ++neteq_set_minimum_delay_events_it_; } - return net_eq_set_minimum_delay; } bool ended() const override { return !NextEventTime(); } @@ -332,7 +282,6 @@ class NetEqStreamInput : public test::NetEqInput { const std::vector::const_iterator neteq_set_minimum_delay_events_end_; const absl::optional end_time_ms_; - std::unique_ptr event_; }; namespace { diff --git a/test/fuzzers/neteq_rtp_fuzzer.cc b/test/fuzzers/neteq_rtp_fuzzer.cc index afde2a87c7..3caa5fe5de 100644 --- a/test/fuzzers/neteq_rtp_fuzzer.cc +++ b/test/fuzzers/neteq_rtp_fuzzer.cc @@ -64,65 +64,71 @@ class FuzzRtpInput : public NetEqInput { new SineGenerator(config.sample_rate_hz)); input_.reset(new EncodeNetEqInput(std::move(generator), std::move(encoder), std::numeric_limits::max())); - // We pop the first event so we have information about the timing of such - // first event. - event_ = input_->PopEvent(); - if (event_ && event_->type() == Event::Type::kPacketData) { - FuzzPacket(static_cast(event_.get())); - } + packet_ = input_->PopPacket(); + FuzzHeader(); + MaybeFuzzPayload(); } - std::unique_ptr PopEvent() override { - std::unique_ptr event_to_return = std::move(event_); - event_ = input_->PopEvent(); - if (event_ && event_->type() == Event::Type::kPacketData) { - FuzzPacket(static_cast(event_.get())); - } - return event_to_return; + absl::optional NextPacketTime() const override { + return packet_->time_ms; + } + + absl::optional NextOutputEventTime() const override { + return input_->NextOutputEventTime(); + } + + absl::optional NextSetMinimumDelayInfo() const override { + return input_->NextSetMinimumDelayInfo(); + } + + std::unique_ptr PopPacket() override { + RTC_DCHECK(packet_); + std::unique_ptr packet_to_return = std::move(packet_); + packet_ = input_->PopPacket(); + FuzzHeader(); + MaybeFuzzPayload(); + return packet_to_return; + } + + void AdvanceOutputEvent() override { return input_->AdvanceOutputEvent(); } + + void AdvanceSetMinimumDelay() override { + return input_->AdvanceSetMinimumDelay(); } bool ended() const override { return ended_; } absl::optional NextHeader() const override { - RTC_DCHECK(event_ && event_->type() == Event::Type::kPacketData); - return static_cast(*event_).header; - } - - absl::optional NextEventTime() const override { - if (event_) { - return event_->timestamp_ms(); - } - return absl::nullopt; + RTC_DCHECK(packet_); + return packet_->header; } private: - void FuzzPacket(PacketData* packet_data) { - FuzzHeader(packet_data->header); - MaybeFuzzPayload(packet_data->payload); - } - - void FuzzHeader(RTPHeader& header) { + void FuzzHeader() { constexpr size_t kNumBytesToFuzz = 11; if (data_ix_ + kNumBytesToFuzz > data_.size()) { ended_ = true; return; } + RTC_DCHECK(packet_); const size_t start_ix = data_ix_; - header.payloadType = + packet_->header.payloadType = ByteReader::ReadLittleEndian(&data_[data_ix_]); - header.payloadType &= 0x7F; + packet_->header.payloadType &= 0x7F; data_ix_ += sizeof(uint8_t); - header.sequenceNumber = + packet_->header.sequenceNumber = ByteReader::ReadLittleEndian(&data_[data_ix_]); data_ix_ += sizeof(uint16_t); - header.timestamp = ByteReader::ReadLittleEndian(&data_[data_ix_]); + packet_->header.timestamp = + ByteReader::ReadLittleEndian(&data_[data_ix_]); data_ix_ += sizeof(uint32_t); - header.ssrc = ByteReader::ReadLittleEndian(&data_[data_ix_]); + packet_->header.ssrc = + ByteReader::ReadLittleEndian(&data_[data_ix_]); data_ix_ += sizeof(uint32_t); RTC_CHECK_EQ(data_ix_ - start_ix, kNumBytesToFuzz); } - void MaybeFuzzPayload(rtc::Buffer& payload) { + void MaybeFuzzPayload() { // Read one byte of fuzz data to determine how many payload bytes to fuzz. if (data_ix_ + 1 > data_.size()) { ended_ = true; @@ -132,7 +138,7 @@ class FuzzRtpInput : public NetEqInput { // Restrict number of bytes to fuzz to 16; a reasonably low number enough to // cover a few RED headers. Also don't write outside the payload length. - bytes_to_fuzz = std::min(bytes_to_fuzz % 16, payload.size()); + bytes_to_fuzz = std::min(bytes_to_fuzz % 16, packet_->payload.size()); if (bytes_to_fuzz == 0) return; @@ -142,7 +148,7 @@ class FuzzRtpInput : public NetEqInput { return; } - std::memcpy(payload.data(), &data_[data_ix_], bytes_to_fuzz); + std::memcpy(packet_->payload.data(), &data_[data_ix_], bytes_to_fuzz); data_ix_ += bytes_to_fuzz; } @@ -150,7 +156,7 @@ class FuzzRtpInput : public NetEqInput { rtc::ArrayView data_; size_t data_ix_ = 0; std::unique_ptr input_; - std::unique_ptr event_; + std::unique_ptr packet_; }; } // namespace diff --git a/test/fuzzers/neteq_signal_fuzzer.cc b/test/fuzzers/neteq_signal_fuzzer.cc index 6ca8c7dfaf..485c38085e 100644 --- a/test/fuzzers/neteq_signal_fuzzer.cc +++ b/test/fuzzers/neteq_signal_fuzzer.cc @@ -80,10 +80,8 @@ class FuzzSignalInput : public NetEqInput { new SineAndNoiseGenerator(config.sample_rate_hz, fuzz_data)); input_.reset(new EncodeNetEqInput(std::move(generator), std::move(encoder), std::numeric_limits::max())); - // We pop the first event so we have information about the timing of such - // first event. - PopPacket(); - PopEvent(); + packet_ = input_->PopPacket(); + // Select an output event period. This is how long time we wait between each // call to NetEq::GetAudio. 10 ms is nominal, 9 and 11 ms will both lead to // clock drift (in different directions). @@ -91,45 +89,32 @@ class FuzzSignalInput : public NetEqInput { output_event_period_ms_ = fuzz_data_.SelectOneOf(output_event_periods); } - std::unique_ptr PopEvent() override { - std::unique_ptr event_to_return = std::move(event_); - if (packet_ && packet_->timestamp_ms() <= next_output_event_ms_) { - event_ = PopPacket(); - } else { - event_ = std::make_unique(next_output_event_ms_); - next_output_event_ms_ += output_event_period_ms_; - } - return event_to_return; + absl::optional NextPacketTime() const override { + return packet_->time_ms; } - absl::optional NextEventTime() const override { - if (event_) { - return event_->timestamp_ms(); - } - return absl::nullopt; + absl::optional NextOutputEventTime() const override { + return next_output_event_ms_; } - std::unique_ptr PopPacket() { - std::unique_ptr packet_to_return = std::move(packet_); - int64_t packet_to_return_time_ms = - packet_to_return ? packet_to_return->timestamp_ms() : 0; - PacketData* packet_data = nullptr; + absl::optional NextSetMinimumDelayInfo() const override { + return input_->NextSetMinimumDelayInfo(); + } + + std::unique_ptr PopPacket() override { + RTC_DCHECK(packet_); + std::unique_ptr packet_to_return = std::move(packet_); do { - std::unique_ptr event = input_->PopEvent(); - while (event && event->type() != Event::Type::kPacketData) { - event = input_->PopEvent(); - } - packet_data = static_cast(event.get()); + packet_ = input_->PopPacket(); // If the next value from the fuzzer input is 0, the packet is discarded // and the next one is pulled from the source. } while (fuzz_data_.CanReadBytes(1) && fuzz_data_.Read() == 0); - if (fuzz_data_.CanReadBytes(1) && packet_data) { + if (fuzz_data_.CanReadBytes(1)) { // Generate jitter by setting an offset for the arrival time. const int8_t arrival_time_offset_ms = fuzz_data_.Read(); // The arrival time can not be before the previous packets. - packet_data->timestamp_ms_ = - std::max(packet_to_return_time_ms, - packet_data->timestamp_ms_ + arrival_time_offset_ms); + packet_->time_ms = std::max(packet_to_return->time_ms, + packet_->time_ms + arrival_time_offset_ms); } else { // Mark that we are at the end of the test. However, the current packet is // still valid (but it may not have been fuzzed as expected). @@ -138,21 +123,28 @@ class FuzzSignalInput : public NetEqInput { return packet_to_return; } + void AdvanceOutputEvent() override { + next_output_event_ms_ += output_event_period_ms_; + } + + void AdvanceSetMinimumDelay() override { + return input_->AdvanceSetMinimumDelay(); + } + bool ended() const override { return ended_; } absl::optional NextHeader() const override { RTC_DCHECK(packet_); - return static_cast(*packet_).header; + return packet_->header; } private: bool ended_ = false; FuzzDataHelper& fuzz_data_; std::unique_ptr input_; - std::unique_ptr packet_; + std::unique_ptr packet_; int64_t next_output_event_ms_ = 0; - int64_t output_event_period_ms_; - std::unique_ptr event_; + int64_t output_event_period_ms_ = 10; }; template