/* * Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. * * Use of this source code is governed by a BSD-style license * that can be found in the LICENSE file in the root of the source * tree. An additional intellectual property rights grant can be found * in the file PATENTS. All contributing project authors may * be found in the AUTHORS file in the root of the source tree. */ #include "modules/audio_processing/aec3/echo_remover.h" #include #include #include #include #include #include "api/array_view.h" #include "modules/audio_processing/aec3/aec3_common.h" #include "modules/audio_processing/aec3/aec_state.h" #include "modules/audio_processing/aec3/comfort_noise_generator.h" #include "modules/audio_processing/aec3/echo_path_variability.h" #include "modules/audio_processing/aec3/echo_remover_metrics.h" #include "modules/audio_processing/aec3/fft_data.h" #include "modules/audio_processing/aec3/output_selector.h" #include "modules/audio_processing/aec3/render_buffer.h" #include "modules/audio_processing/aec3/render_delay_buffer.h" #include "modules/audio_processing/aec3/residual_echo_estimator.h" #include "modules/audio_processing/aec3/subtractor.h" #include "modules/audio_processing/aec3/suppression_filter.h" #include "modules/audio_processing/aec3/suppression_gain.h" #include "modules/audio_processing/logging/apm_data_dumper.h" #include "rtc_base/atomicops.h" #include "rtc_base/constructormagic.h" namespace webrtc { namespace { void LinearEchoPower(const FftData& E, const FftData& Y, std::array* S2) { for (size_t k = 0; k < E.re.size(); ++k) { (*S2)[k] = (Y.re[k] - E.re[k]) * (Y.re[k] - E.re[k]) + (Y.im[k] - E.im[k]) * (Y.im[k] - E.im[k]); } } // Class for removing the echo from the capture signal. class EchoRemoverImpl final : public EchoRemover { public: explicit EchoRemoverImpl(const EchoCanceller3Config& config, int sample_rate_hz); ~EchoRemoverImpl() override; // Removes the echo from a block of samples from the capture signal. The // supplied render signal is assumed to be pre-aligned with the capture // signal. void ProcessCapture(const rtc::Optional& echo_path_delay_samples, const EchoPathVariability& echo_path_variability, bool capture_signal_saturation, const RenderBuffer& render_buffer, std::vector>* capture) override; // Updates the status on whether echo leakage is detected in the output of the // echo remover. void UpdateEchoLeakageStatus(bool leakage_detected) override { echo_leakage_detected_ = leakage_detected; } private: static int instance_count_; const EchoCanceller3Config config_; const Aec3Fft fft_; std::unique_ptr data_dumper_; const Aec3Optimization optimization_; const int sample_rate_hz_; Subtractor subtractor_; SuppressionGain suppression_gain_; ComfortNoiseGenerator cng_; SuppressionFilter suppression_filter_; RenderSignalAnalyzer render_signal_analyzer_; OutputSelector output_selector_; ResidualEchoEstimator residual_echo_estimator_; bool echo_leakage_detected_ = false; AecState aec_state_; EchoRemoverMetrics metrics_; RTC_DISALLOW_COPY_AND_ASSIGN(EchoRemoverImpl); }; int EchoRemoverImpl::instance_count_ = 0; EchoRemoverImpl::EchoRemoverImpl(const EchoCanceller3Config& config, int sample_rate_hz) : config_(config), fft_(), data_dumper_( new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))), optimization_(DetectOptimization()), sample_rate_hz_(sample_rate_hz), subtractor_(data_dumper_.get(), optimization_), suppression_gain_(config_, optimization_), cng_(optimization_), suppression_filter_(sample_rate_hz_), residual_echo_estimator_(config_), aec_state_(config_) { RTC_DCHECK(ValidFullBandRate(sample_rate_hz)); } EchoRemoverImpl::~EchoRemoverImpl() = default; void EchoRemoverImpl::ProcessCapture( const rtc::Optional& echo_path_delay_samples, const EchoPathVariability& echo_path_variability, bool capture_signal_saturation, const RenderBuffer& render_buffer, std::vector>* capture) { const std::vector>& x = render_buffer.MostRecentBlock(); std::vector>* y = capture; RTC_DCHECK(y); RTC_DCHECK_EQ(x.size(), NumBandsForRate(sample_rate_hz_)); RTC_DCHECK_EQ(y->size(), NumBandsForRate(sample_rate_hz_)); RTC_DCHECK_EQ(x[0].size(), kBlockSize); RTC_DCHECK_EQ((*y)[0].size(), kBlockSize); const std::vector& x0 = x[0]; std::vector& y0 = (*y)[0]; data_dumper_->DumpWav("aec3_echo_remover_capture_input", kBlockSize, &y0[0], LowestBandRate(sample_rate_hz_), 1); data_dumper_->DumpWav("aec3_echo_remover_render_input", kBlockSize, &x0[0], LowestBandRate(sample_rate_hz_), 1); data_dumper_->DumpRaw("aec3_echo_remover_capture_input", y0); data_dumper_->DumpRaw("aec3_echo_remover_render_input", x0); aec_state_.UpdateCaptureSaturation(capture_signal_saturation); if (echo_path_variability.AudioPathChanged()) { subtractor_.HandleEchoPathChange(echo_path_variability); aec_state_.HandleEchoPathChange(echo_path_variability); } std::array Y2; std::array R2; std::array S2_linear; std::array G; float high_bands_gain; FftData Y; FftData comfort_noise; FftData high_band_comfort_noise; SubtractorOutput subtractor_output; FftData& E_main = subtractor_output.E_main; auto& E2_main = subtractor_output.E2_main; auto& E2_shadow = subtractor_output.E2_shadow; auto& e_main = subtractor_output.e_main; // Analyze the render signal. render_signal_analyzer_.Update(render_buffer, aec_state_.FilterDelay()); // Perform linear echo cancellation. subtractor_.Process(render_buffer, y0, render_signal_analyzer_, aec_state_, &subtractor_output); // Compute spectra. fft_.ZeroPaddedFft(y0, &Y); LinearEchoPower(E_main, Y, &S2_linear); Y.Spectrum(optimization_, &Y2); // Update the AEC state information. aec_state_.Update(subtractor_.FilterFrequencyResponse(), subtractor_.FilterImpulseResponse(), subtractor_.ConvergedFilter(), echo_path_delay_samples, render_buffer, E2_main, Y2, x0, subtractor_output.s_main, echo_leakage_detected_); // Choose the linear output. output_selector_.FormLinearOutput(!aec_state_.TransparentMode(), e_main, y0); data_dumper_->DumpWav("aec3_output_linear", kBlockSize, &y0[0], LowestBandRate(sample_rate_hz_), 1); data_dumper_->DumpRaw("aec3_output_linear", y0); const auto& E2 = output_selector_.UseSubtractorOutput() ? E2_main : Y2; // Estimate the residual echo power. residual_echo_estimator_.Estimate(aec_state_, render_buffer, S2_linear, Y2, &R2); // Estimate the comfort noise. cng_.Compute(aec_state_, Y2, &comfort_noise, &high_band_comfort_noise); // A choose and apply echo suppression gain. suppression_gain_.GetGain(E2, R2, cng_.NoiseSpectrum(), render_signal_analyzer_, aec_state_, x, &high_bands_gain, &G); suppression_filter_.ApplyGain(comfort_noise, high_band_comfort_noise, G, high_bands_gain, y); // Update the metrics. metrics_.Update(aec_state_, cng_.NoiseSpectrum(), G); // Update the aec state with the aec output characteristics. aec_state_.UpdateWithOutput(y0); // Debug outputs for the purpose of development and analysis. data_dumper_->DumpWav("aec3_echo_estimate", kBlockSize, &subtractor_output.s_main[0], LowestBandRate(sample_rate_hz_), 1); data_dumper_->DumpRaw("aec3_output", y0); data_dumper_->DumpRaw("aec3_narrow_render", render_signal_analyzer_.NarrowPeakBand() ? 1 : 0); data_dumper_->DumpRaw("aec3_N2", cng_.NoiseSpectrum()); data_dumper_->DumpRaw("aec3_suppressor_gain", G); data_dumper_->DumpWav("aec3_output", rtc::ArrayView(&y0[0], kBlockSize), LowestBandRate(sample_rate_hz_), 1); data_dumper_->DumpRaw("aec3_using_subtractor_output", output_selector_.UseSubtractorOutput() ? 1 : 0); data_dumper_->DumpRaw("aec3_E2", E2); data_dumper_->DumpRaw("aec3_E2_main", E2_main); data_dumper_->DumpRaw("aec3_E2_shadow", E2_shadow); data_dumper_->DumpRaw("aec3_S2_linear", S2_linear); data_dumper_->DumpRaw("aec3_Y2", Y2); data_dumper_->DumpRaw("aec3_X2", render_buffer.Spectrum(0)); data_dumper_->DumpRaw("aec3_R2", R2); data_dumper_->DumpRaw("aec3_erle", aec_state_.Erle()); data_dumper_->DumpRaw("aec3_erl", aec_state_.Erl()); data_dumper_->DumpRaw("aec3_active_render", aec_state_.ActiveRender()); data_dumper_->DumpRaw("aec3_usable_linear_estimate", aec_state_.UsableLinearEstimate()); data_dumper_->DumpRaw( "aec3_filter_delay", aec_state_.FilterDelay() ? *aec_state_.FilterDelay() : -1); data_dumper_->DumpRaw( "aec3_external_delay", aec_state_.ExternalDelay() ? *aec_state_.ExternalDelay() : -1); data_dumper_->DumpRaw("aec3_capture_saturation", aec_state_.SaturatedCapture() ? 1 : 0); } } // namespace EchoRemover* EchoRemover::Create(const EchoCanceller3Config& config, int sample_rate_hz) { return new EchoRemoverImpl(config, sample_rate_hz); } } // namespace webrtc