/* * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "modules/audio_processing/aec3/render_delay_buffer.h" #include #include #include "modules/audio_processing/aec3/aec3_common.h" #include "modules/audio_processing/aec3/aec3_fft.h" #include "modules/audio_processing/aec3/block_processor.h" #include "modules/audio_processing/aec3/decimator.h" #include "modules/audio_processing/aec3/fft_buffer.h" #include "modules/audio_processing/aec3/fft_data.h" #include "modules/audio_processing/aec3/matrix_buffer.h" #include "rtc_base/atomicops.h" #include "rtc_base/checks.h" #include "rtc_base/constructormagic.h" #include "rtc_base/logging.h" namespace webrtc { namespace { class RenderDelayBufferImpl final : public RenderDelayBuffer { public: RenderDelayBufferImpl(const EchoCanceller3Config& config, size_t num_bands); ~RenderDelayBufferImpl() override; void Reset() override; BufferingEvent Insert(const std::vector>& block) override; BufferingEvent PrepareCaptureProcessing() override; bool SetDelay(size_t delay) override; rtc::Optional Delay() const override { return delay_; } size_t MaxDelay() const override { return blocks_.buffer.size() - 1 - buffer_headroom_; } RenderBuffer* GetRenderBuffer() override { return &echo_remover_buffer_; } const DownsampledRenderBuffer& GetDownsampledRenderBuffer() const override { return low_rate_; } bool CausalDelay() const override; private: static int instance_count_; std::unique_ptr data_dumper_; const Aec3Optimization optimization_; const EchoCanceller3Config config_; const int sub_block_size_; MatrixBuffer blocks_; VectorBuffer spectra_; FftBuffer ffts_; rtc::Optional delay_; rtc::Optional internal_delay_; RenderBuffer echo_remover_buffer_; DownsampledRenderBuffer low_rate_; Decimator render_decimator_; const std::vector> zero_block_; const Aec3Fft fft_; std::vector render_ds_; const int buffer_headroom_; int LowRateBufferOffset() const { return DelayEstimatorOffset(config_) >> 1; } int MaxExternalDelayToInternalDelay(size_t delay) const; void ApplyDelay(int delay); void InsertBlock(const std::vector>& block, int previous_write); RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayBufferImpl); }; // Increases the write indices for the render buffers. void IncreaseWriteIndices(int sub_block_size, MatrixBuffer* blocks, VectorBuffer* spectra, FftBuffer* ffts, DownsampledRenderBuffer* low_rate) { low_rate->UpdateWriteIndex(-sub_block_size); blocks->IncWriteIndex(); spectra->DecWriteIndex(); ffts->DecWriteIndex(); } // Increases the read indices for the render buffers. void IncreaseReadIndices(const rtc::Optional& delay, int sub_block_size, MatrixBuffer* blocks, VectorBuffer* spectra, FftBuffer* ffts, DownsampledRenderBuffer* low_rate) { RTC_DCHECK_NE(low_rate->read, low_rate->write); low_rate->UpdateReadIndex(-sub_block_size); if (blocks->read != blocks->write) { blocks->IncReadIndex(); spectra->DecReadIndex(); ffts->DecReadIndex(); } else { // Only allow underrun for blocks_ when the delay is not set. RTC_DCHECK(!delay); } } // Checks for a render buffer overrun. bool RenderOverrun(const MatrixBuffer& b, const DownsampledRenderBuffer& l) { return l.read == l.write || b.read == b.write; } // Checks for a render buffer underrun. If the delay is not specified, only the // low rate buffer underrun is counted as the delay offset for the other buffers // is unknown. bool RenderUnderrun(const rtc::Optional& delay, const MatrixBuffer& b, const DownsampledRenderBuffer& l) { return l.read == l.write || (delay && b.read == b.write); } // Computes the latency in the buffer (the number of unread elements). int BufferLatency(const DownsampledRenderBuffer& l) { return (l.buffer.size() + l.read - l.write) % l.buffer.size(); } // Computes the mismatch between the number of render and capture calls based on // the known offset (achieved during reset) of the low rate buffer. bool ApiCallSkew(const DownsampledRenderBuffer& low_rate_buffer, int sub_block_size, int low_rate_buffer_offset_sub_blocks) { int latency = BufferLatency(low_rate_buffer); int skew = abs(low_rate_buffer_offset_sub_blocks * sub_block_size - latency); int skew_limit = low_rate_buffer_offset_sub_blocks * sub_block_size; return skew >= skew_limit; } int RenderDelayBufferImpl::instance_count_ = 0; RenderDelayBufferImpl::RenderDelayBufferImpl(const EchoCanceller3Config& config, size_t num_bands) : data_dumper_( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), optimization_(DetectOptimization()), config_(config), sub_block_size_( static_cast(config.delay.down_sampling_factor > 0 ? kBlockSize / config.delay.down_sampling_factor : kBlockSize)), blocks_(GetRenderDelayBufferSize(config.delay.down_sampling_factor, config.delay.num_filters, config.filter.length_blocks), num_bands, kBlockSize), spectra_(blocks_.buffer.size(), kFftLengthBy2Plus1), ffts_(blocks_.buffer.size()), delay_(config_.delay.min_echo_path_delay_blocks), echo_remover_buffer_(config.filter.length_blocks, &blocks_, &spectra_, &ffts_), low_rate_(GetDownSampledBufferSize(config.delay.down_sampling_factor, config.delay.num_filters)), render_decimator_(config.delay.down_sampling_factor), zero_block_(num_bands, std::vector(kBlockSize, 0.f)), fft_(), render_ds_(sub_block_size_, 0.f), buffer_headroom_(config.filter.length_blocks) { RTC_DCHECK_EQ(blocks_.buffer.size(), ffts_.buffer.size()); RTC_DCHECK_EQ(spectra_.buffer.size(), ffts_.buffer.size()); // Necessary condition to avoid unrecoverable echp due to noncausal alignment. RTC_DCHECK_EQ(DelayEstimatorOffset(config_), LowRateBufferOffset() * 2); Reset(); } RenderDelayBufferImpl::~RenderDelayBufferImpl() = default; // Resets the buffer delays and clears the reported delays. void RenderDelayBufferImpl::Reset() { // Pre-fill the low rate buffer (which is used for delay estimation) to add // headroom for the allowed api call jitter. low_rate_.read = low_rate_.OffsetIndex( low_rate_.write, LowRateBufferOffset() * sub_block_size_); // Set the render buffer delays to the default delay. ApplyDelay(config_.delay.default_delay); // Unset the delays which are set by ApplyConfig. delay_ = rtc::nullopt; internal_delay_ = rtc::nullopt; } // Inserts a new block into the render buffers. RenderDelayBuffer::BufferingEvent RenderDelayBufferImpl::Insert( const std::vector>& block) { // Increase the write indices to where the new blocks should be written. const int previous_write = blocks_.write; IncreaseWriteIndices(sub_block_size_, &blocks_, &spectra_, &ffts_, &low_rate_); // Allow overrun and do a reset when render overrun occurrs due to more render // data being inserted than capture data is received. BufferingEvent event = RenderOverrun(blocks_, low_rate_) ? event = BufferingEvent::kRenderOverrun : BufferingEvent::kNone; // Insert the new render block into the specified position. InsertBlock(block, previous_write); if (event != BufferingEvent::kNone) { Reset(); } return event; } // Prepares the render buffers for processing another capture block. RenderDelayBuffer::BufferingEvent RenderDelayBufferImpl::PrepareCaptureProcessing() { BufferingEvent event = BufferingEvent::kNone; if (RenderUnderrun(internal_delay_, blocks_, low_rate_)) { // Don't increase the read indices if there is a render underrun. event = BufferingEvent::kRenderUnderrun; } else { // Increase the read indices in the render buffers to point to the most // recent block to use in the capture processing. IncreaseReadIndices(internal_delay_, sub_block_size_, &blocks_, &spectra_, &ffts_, &low_rate_); // Check for skew in the API calls which, if too large, causes the delay // estimation to be noncausal. Doing this check after the render indice // increase saves one unit of allowed skew. Note that the skew check only // should need to be one-sided as one of the skew directions results in an // underrun. bool skew = ApiCallSkew(low_rate_, sub_block_size_, LowRateBufferOffset()); event = skew ? BufferingEvent::kApiCallSkew : BufferingEvent::kNone; } if (event != BufferingEvent::kNone) { Reset(); } return event; } // Sets the delay and returns a bool indicating whether the delay was changed. bool RenderDelayBufferImpl::SetDelay(size_t delay) { if (delay_ && *delay_ == delay) { return false; } delay_ = delay; // Compute the internal delay and limit the delay to the allowed range. int internal_delay = MaxExternalDelayToInternalDelay(*delay_); internal_delay_ = std::min(MaxDelay(), static_cast(std::max(internal_delay, 0))); // Apply the delay to the buffers. ApplyDelay(*internal_delay_); return true; } // Returns whether the specified delay is causal. bool RenderDelayBufferImpl::CausalDelay() const { return !internal_delay_ || *internal_delay_ >= static_cast(config_.delay.min_echo_path_delay_blocks); } // Maps the externally computed delay to the delay used internally. int RenderDelayBufferImpl::MaxExternalDelayToInternalDelay( size_t external_delay_blocks) const { const int latency = BufferLatency(low_rate_); RTC_DCHECK_LT(0, sub_block_size_); RTC_DCHECK_EQ(0, latency % sub_block_size_); int latency_blocks = latency / sub_block_size_; return latency_blocks + static_cast(external_delay_blocks) - DelayEstimatorOffset(config_); } // Set the read indices according to the delay. void RenderDelayBufferImpl::ApplyDelay(int delay) { blocks_.read = blocks_.OffsetIndex(blocks_.write, -delay); spectra_.read = spectra_.OffsetIndex(spectra_.write, delay); ffts_.read = ffts_.OffsetIndex(ffts_.write, delay); } // Inserts a block into the render buffers. void RenderDelayBufferImpl::InsertBlock( const std::vector>& block, int previous_write) { auto& b = blocks_; auto& lr = low_rate_; auto& ds = render_ds_; auto& f = ffts_; auto& s = spectra_; RTC_DCHECK_EQ(block.size(), b.buffer[b.write].size()); for (size_t k = 0; k < block.size(); ++k) { RTC_DCHECK_EQ(block[k].size(), b.buffer[b.write][k].size()); std::copy(block[k].begin(), block[k].end(), b.buffer[b.write][k].begin()); } render_decimator_.Decimate(block[0], ds); std::copy(ds.rbegin(), ds.rend(), lr.buffer.begin() + lr.write); fft_.PaddedFft(block[0], b.buffer[previous_write][0], &f.buffer[f.write]); f.buffer[f.write].Spectrum(optimization_, s.buffer[s.write]); } } // namespace int RenderDelayBuffer::RenderDelayBuffer::DelayEstimatorOffset( const EchoCanceller3Config& config) { return config.delay.api_call_jitter_blocks * 2; } RenderDelayBuffer* RenderDelayBuffer::Create(const EchoCanceller3Config& config, size_t num_bands) { return new RenderDelayBufferImpl(config, num_bands); } } // namespace webrtc