/* * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "modules/audio_processing/aec3/render_delay_buffer.h" #include #include #include "modules/audio_processing/aec3/aec3_common.h" #include "modules/audio_processing/aec3/block_processor.h" #include "modules/audio_processing/aec3/decimator.h" #include "modules/audio_processing/aec3/fft_data.h" #include "rtc_base/atomicops.h" #include "rtc_base/checks.h" #include "rtc_base/constructormagic.h" #include "rtc_base/logging.h" namespace webrtc { namespace { class ApiCallJitterBuffer { public: explicit ApiCallJitterBuffer(size_t num_bands) { buffer_.fill(std::vector>( num_bands, std::vector(kBlockSize, 0.f))); } ~ApiCallJitterBuffer() = default; void Reset() { size_ = 0; last_insert_index_ = 0; } void Insert(const std::vector>& block) { RTC_DCHECK_LT(size_, buffer_.size()); last_insert_index_ = (last_insert_index_ + 1) % buffer_.size(); RTC_DCHECK_EQ(buffer_[last_insert_index_].size(), block.size()); RTC_DCHECK_EQ(buffer_[last_insert_index_][0].size(), block[0].size()); for (size_t k = 0; k < block.size(); ++k) { std::copy(block[k].begin(), block[k].end(), buffer_[last_insert_index_][k].begin()); } ++size_; } void Remove(std::vector>* block) { RTC_DCHECK_LT(0, size_); --size_; const size_t extract_index = (last_insert_index_ - size_ + buffer_.size()) % buffer_.size(); for (size_t k = 0; k < block->size(); ++k) { std::copy(buffer_[extract_index][k].begin(), buffer_[extract_index][k].end(), (*block)[k].begin()); } } size_t Size() const { return size_; } bool Full() const { return size_ >= (buffer_.size()); } bool Empty() const { return size_ == 0; } private: std::array>, kMaxApiCallsJitterBlocks> buffer_; size_t size_ = 0; int last_insert_index_ = 0; }; class RenderDelayBufferImpl final : public RenderDelayBuffer { public: RenderDelayBufferImpl(size_t num_bands, size_t down_sampling_factor, size_t downsampled_render_buffer_size, size_t render_delay_buffer_size); ~RenderDelayBufferImpl() override; void Reset() override; bool Insert(const std::vector>& block) override; bool UpdateBuffers() override; void SetDelay(size_t delay) override; size_t Delay() const override { return delay_; } const RenderBuffer& GetRenderBuffer() const override { return fft_buffer_; } const DownsampledRenderBuffer& GetDownsampledRenderBuffer() const override { return downsampled_render_buffer_; } private: static int instance_count_; std::unique_ptr data_dumper_; const Aec3Optimization optimization_; const size_t down_sampling_factor_; const size_t sub_block_size_; std::vector>> buffer_; size_t delay_ = 0; size_t last_insert_index_ = 0; RenderBuffer fft_buffer_; DownsampledRenderBuffer downsampled_render_buffer_; Decimator render_decimator_; ApiCallJitterBuffer api_call_jitter_buffer_; const std::vector> zero_block_; RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(RenderDelayBufferImpl); }; int RenderDelayBufferImpl::instance_count_ = 0; RenderDelayBufferImpl::RenderDelayBufferImpl( size_t num_bands, size_t down_sampling_factor, size_t downsampled_render_buffer_size, size_t render_delay_buffer_size) : data_dumper_( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), optimization_(DetectOptimization()), down_sampling_factor_(down_sampling_factor), sub_block_size_(down_sampling_factor_ > 0 ? kBlockSize / down_sampling_factor : kBlockSize), buffer_( render_delay_buffer_size, std::vector>(num_bands, std::vector(kBlockSize, 0.f))), fft_buffer_( optimization_, num_bands, std::max(kUnknownDelayRenderWindowSize, kAdaptiveFilterLength), std::vector(1, kAdaptiveFilterLength)), downsampled_render_buffer_(downsampled_render_buffer_size), render_decimator_(down_sampling_factor_), api_call_jitter_buffer_(num_bands), zero_block_(num_bands, std::vector(kBlockSize, 0.f)) { RTC_DCHECK_LT(buffer_.size(), downsampled_render_buffer_.buffer.size()); } RenderDelayBufferImpl::~RenderDelayBufferImpl() = default; void RenderDelayBufferImpl::Reset() { // Empty all data in the buffers. delay_ = 0; last_insert_index_ = 0; downsampled_render_buffer_.position = 0; std::fill(downsampled_render_buffer_.buffer.begin(), downsampled_render_buffer_.buffer.end(), 0.f); fft_buffer_.Clear(); api_call_jitter_buffer_.Reset(); for (auto& c : buffer_) { for (auto& b : c) { std::fill(b.begin(), b.end(), 0.f); } } } bool RenderDelayBufferImpl::Insert( const std::vector>& block) { RTC_DCHECK_EQ(block.size(), buffer_[0].size()); RTC_DCHECK_EQ(block[0].size(), buffer_[0][0].size()); if (api_call_jitter_buffer_.Full()) { // Report buffer overrun and let the caller handle the overrun. return false; } api_call_jitter_buffer_.Insert(block); return true; } bool RenderDelayBufferImpl::UpdateBuffers() { bool underrun = true; // Update the buffers with a new block if such is available, otherwise insert // a block of silence. if (api_call_jitter_buffer_.Size() > 0) { last_insert_index_ = (last_insert_index_ + 1) % buffer_.size(); api_call_jitter_buffer_.Remove(&buffer_[last_insert_index_]); underrun = false; } downsampled_render_buffer_.position = (downsampled_render_buffer_.position - sub_block_size_ + downsampled_render_buffer_.buffer.size()) % downsampled_render_buffer_.buffer.size(); rtc::ArrayView input( underrun ? zero_block_[0].data() : buffer_[last_insert_index_][0].data(), kBlockSize); rtc::ArrayView output(downsampled_render_buffer_.buffer.data() + downsampled_render_buffer_.position, sub_block_size_); data_dumper_->DumpWav("aec3_render_decimator_input", input.size(), input.data(), 16000, 1); render_decimator_.Decimate(input, output); data_dumper_->DumpWav("aec3_render_decimator_output", output.size(), output.data(), 16000 / down_sampling_factor_, 1); for (size_t k = 0; k < output.size() / 2; ++k) { float tmp = output[k]; output[k] = output[output.size() - 1 - k]; output[output.size() - 1 - k] = tmp; } if (underrun) { fft_buffer_.Insert(zero_block_); } else { fft_buffer_.Insert(buffer_[(last_insert_index_ - delay_ + buffer_.size()) % buffer_.size()]); } return !underrun; } void RenderDelayBufferImpl::SetDelay(size_t delay) { if (delay_ == delay) { return; } // If there is a new delay set, clear the fft buffer. fft_buffer_.Clear(); if ((buffer_.size() - 1) < delay) { // If the desired delay is larger than the delay buffer, shorten the delay // buffer size to achieve the desired alignment with the available buffer // size. downsampled_render_buffer_.position = (downsampled_render_buffer_.position + sub_block_size_ * (delay - (buffer_.size() - 1))) % downsampled_render_buffer_.buffer.size(); last_insert_index_ = (last_insert_index_ - (delay - (buffer_.size() - 1)) + buffer_.size()) % buffer_.size(); delay_ = buffer_.size() - 1; } else { delay_ = delay; } } } // namespace RenderDelayBuffer* RenderDelayBuffer::Create( size_t num_bands, size_t down_sampling_factor, size_t downsampled_render_buffer_size, size_t render_delay_buffer_size) { return new RenderDelayBufferImpl(num_bands, down_sampling_factor, downsampled_render_buffer_size, render_delay_buffer_size); } } // namespace webrtc