mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-14 14:20:45 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions
webrtc/modules/audio_processing/aec3/echo_remover.cc
Per Åhgren 7ddd46386a Balancing the transparency in AEC3 between saturating and low echo paths 
This CL balances the NLP tradeoff in AEC3 to properly handle the cases
when the echo path is so strong that it saturates the echo and when it
is so weak that the echo is very low compared to nearend.

Bug: webrtc:8411, webrtc:8412, chromium:775653
Change-Id: I5aff74dfadd51cac1ce71b1cb935d68a5be6918d
Reviewed-on: https://webrtc-review.googlesource.com/14120
Commit-Queue: Per Åhgren <peah@webrtc.org>
Reviewed-by: Per Åhgren <peah@webrtc.org>
Reviewed-by: Gustaf Ullberg <gustaf@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#20418}
2017-10-25 01:36:59 +00:00

247 lines
9.8 KiB
C++
Raw Blame History

/*
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/audio_processing/aec3/echo_remover.h"
#include <math.h>
#include <algorithm>
#include <memory>
#include <numeric>
#include <string>
#include "api/array_view.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/aec_state.h"
#include "modules/audio_processing/aec3/comfort_noise_generator.h"
#include "modules/audio_processing/aec3/echo_path_variability.h"
#include "modules/audio_processing/aec3/echo_remover_metrics.h"
#include "modules/audio_processing/aec3/fft_data.h"
#include "modules/audio_processing/aec3/output_selector.h"
#include "modules/audio_processing/aec3/render_buffer.h"
#include "modules/audio_processing/aec3/render_delay_buffer.h"
#include "modules/audio_processing/aec3/residual_echo_estimator.h"
#include "modules/audio_processing/aec3/subtractor.h"
#include "modules/audio_processing/aec3/suppression_filter.h"
#include "modules/audio_processing/aec3/suppression_gain.h"
#include "modules/audio_processing/logging/apm_data_dumper.h"
#include "rtc_base/atomicops.h"
#include "rtc_base/constructormagic.h"
namespace webrtc {
namespace {
void LinearEchoPower(const FftData& E,
const FftData& Y,
std::array<float, kFftLengthBy2Plus1>* S2) {
for (size_t k = 0; k < E.re.size(); ++k) {
(*S2)[k] = (Y.re[k] - E.re[k]) * (Y.re[k] - E.re[k]) +
(Y.im[k] - E.im[k]) * (Y.im[k] - E.im[k]);
}
}
// Class for removing the echo from the capture signal.
class EchoRemoverImpl final : public EchoRemover {
public:
explicit EchoRemoverImpl(const EchoCanceller3Config& config,
int sample_rate_hz);
~EchoRemoverImpl() override;
// Removes the echo from a block of samples from the capture signal. The
// supplied render signal is assumed to be pre-aligned with the capture
// signal.
void ProcessCapture(const rtc::Optional<size_t>& echo_path_delay_samples,
const EchoPathVariability& echo_path_variability,
bool capture_signal_saturation,
const RenderBuffer& render_buffer,
std::vector<std::vector<float>>* capture) override;
// Updates the status on whether echo leakage is detected in the output of the
// echo remover.
void UpdateEchoLeakageStatus(bool leakage_detected) override {
echo_leakage_detected_ = leakage_detected;
}
private:
static int instance_count_;
const EchoCanceller3Config config_;
const Aec3Fft fft_;
std::unique_ptr<ApmDataDumper> data_dumper_;
const Aec3Optimization optimization_;
const int sample_rate_hz_;
Subtractor subtractor_;
SuppressionGain suppression_gain_;
ComfortNoiseGenerator cng_;
SuppressionFilter suppression_filter_;
RenderSignalAnalyzer render_signal_analyzer_;
OutputSelector output_selector_;
ResidualEchoEstimator residual_echo_estimator_;
bool echo_leakage_detected_ = false;
AecState aec_state_;
EchoRemoverMetrics metrics_;
RTC_DISALLOW_COPY_AND_ASSIGN(EchoRemoverImpl);
};
int EchoRemoverImpl::instance_count_ = 0;
EchoRemoverImpl::EchoRemoverImpl(const EchoCanceller3Config& config,
int sample_rate_hz)
: config_(config),
fft_(),
data_dumper_(
new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
optimization_(DetectOptimization()),
sample_rate_hz_(sample_rate_hz),
subtractor_(data_dumper_.get(), optimization_),
suppression_gain_(config_, optimization_),
cng_(optimization_),
suppression_filter_(sample_rate_hz_),
residual_echo_estimator_(config_),
aec_state_(config_) {
RTC_DCHECK(ValidFullBandRate(sample_rate_hz));
}
EchoRemoverImpl::~EchoRemoverImpl() = default;
void EchoRemoverImpl::ProcessCapture(
const rtc::Optional<size_t>& echo_path_delay_samples,
const EchoPathVariability& echo_path_variability,
bool capture_signal_saturation,
const RenderBuffer& render_buffer,
std::vector<std::vector<float>>* capture) {
const std::vector<std::vector<float>>& x = render_buffer.MostRecentBlock();
std::vector<std::vector<float>>* y = capture;
RTC_DCHECK(y);
RTC_DCHECK_EQ(x.size(), NumBandsForRate(sample_rate_hz_));
RTC_DCHECK_EQ(y->size(), NumBandsForRate(sample_rate_hz_));
RTC_DCHECK_EQ(x[0].size(), kBlockSize);
RTC_DCHECK_EQ((*y)[0].size(), kBlockSize);
const std::vector<float>& x0 = x[0];
std::vector<float>& y0 = (*y)[0];
data_dumper_->DumpWav("aec3_echo_remover_capture_input", kBlockSize, &y0[0],
LowestBandRate(sample_rate_hz_), 1);
data_dumper_->DumpWav("aec3_echo_remover_render_input", kBlockSize, &x0[0],
LowestBandRate(sample_rate_hz_), 1);
data_dumper_->DumpRaw("aec3_echo_remover_capture_input", y0);
data_dumper_->DumpRaw("aec3_echo_remover_render_input", x0);
aec_state_.UpdateCaptureSaturation(capture_signal_saturation);
if (echo_path_variability.AudioPathChanged()) {
subtractor_.HandleEchoPathChange(echo_path_variability);
aec_state_.HandleEchoPathChange(echo_path_variability);
}
std::array<float, kFftLengthBy2Plus1> Y2;
std::array<float, kFftLengthBy2Plus1> R2;
std::array<float, kFftLengthBy2Plus1> S2_linear;
std::array<float, kFftLengthBy2Plus1> G;
float high_bands_gain;
FftData Y;
FftData comfort_noise;
FftData high_band_comfort_noise;
SubtractorOutput subtractor_output;
FftData& E_main = subtractor_output.E_main;
auto& E2_main = subtractor_output.E2_main;
auto& E2_shadow = subtractor_output.E2_shadow;
auto& e_main = subtractor_output.e_main;
// Analyze the render signal.
render_signal_analyzer_.Update(render_buffer, aec_state_.FilterDelay());
// Perform linear echo cancellation.
subtractor_.Process(render_buffer, y0, render_signal_analyzer_, aec_state_,
&subtractor_output);
// Compute spectra.
fft_.ZeroPaddedFft(y0, &Y);
LinearEchoPower(E_main, Y, &S2_linear);
Y.Spectrum(optimization_, &Y2);
// Update the AEC state information.
aec_state_.Update(subtractor_.FilterFrequencyResponse(),
subtractor_.FilterImpulseResponse(),
subtractor_.ConvergedFilter(), echo_path_delay_samples,
render_buffer, E2_main, Y2, x0, subtractor_output.s_main,
echo_leakage_detected_);
// Choose the linear output.
output_selector_.FormLinearOutput(!aec_state_.TransparentMode(), e_main, y0);
data_dumper_->DumpWav("aec3_output_linear", kBlockSize, &y0[0],
LowestBandRate(sample_rate_hz_), 1);
data_dumper_->DumpRaw("aec3_output_linear", y0);
const auto& E2 = output_selector_.UseSubtractorOutput() ? E2_main : Y2;
// Estimate the residual echo power.
residual_echo_estimator_.Estimate(aec_state_, render_buffer, S2_linear, Y2,
&R2);
// Estimate the comfort noise.
cng_.Compute(aec_state_, Y2, &comfort_noise, &high_band_comfort_noise);
// A choose and apply echo suppression gain.
suppression_gain_.GetGain(E2, R2, cng_.NoiseSpectrum(),
render_signal_analyzer_, aec_state_, x,
&high_bands_gain, &G);
suppression_filter_.ApplyGain(comfort_noise, high_band_comfort_noise, G,
high_bands_gain, y);
// Update the metrics.
metrics_.Update(aec_state_, cng_.NoiseSpectrum(), G);
// Update the aec state with the aec output characteristics.
aec_state_.UpdateWithOutput(y0);
// Debug outputs for the purpose of development and analysis.
data_dumper_->DumpWav("aec3_echo_estimate", kBlockSize,
&subtractor_output.s_main[0],
LowestBandRate(sample_rate_hz_), 1);
data_dumper_->DumpRaw("aec3_output", y0);
data_dumper_->DumpRaw("aec3_narrow_render",
render_signal_analyzer_.NarrowPeakBand() ? 1 : 0);
data_dumper_->DumpRaw("aec3_N2", cng_.NoiseSpectrum());
data_dumper_->DumpRaw("aec3_suppressor_gain", G);
data_dumper_->DumpWav("aec3_output",
rtc::ArrayView<const float>(&y0[0], kBlockSize),
LowestBandRate(sample_rate_hz_), 1);
data_dumper_->DumpRaw("aec3_using_subtractor_output",
output_selector_.UseSubtractorOutput() ? 1 : 0);
data_dumper_->DumpRaw("aec3_E2", E2);
data_dumper_->DumpRaw("aec3_E2_main", E2_main);
data_dumper_->DumpRaw("aec3_E2_shadow", E2_shadow);
data_dumper_->DumpRaw("aec3_S2_linear", S2_linear);
data_dumper_->DumpRaw("aec3_Y2", Y2);
data_dumper_->DumpRaw("aec3_X2", render_buffer.Spectrum(0));
data_dumper_->DumpRaw("aec3_R2", R2);
data_dumper_->DumpRaw("aec3_erle", aec_state_.Erle());
data_dumper_->DumpRaw("aec3_erl", aec_state_.Erl());
data_dumper_->DumpRaw("aec3_active_render", aec_state_.ActiveRender());
data_dumper_->DumpRaw("aec3_usable_linear_estimate",
aec_state_.UsableLinearEstimate());
data_dumper_->DumpRaw(
"aec3_filter_delay",
aec_state_.FilterDelay() ? *aec_state_.FilterDelay() : -1);
data_dumper_->DumpRaw(
"aec3_external_delay",
aec_state_.ExternalDelay() ? *aec_state_.ExternalDelay() : -1);
data_dumper_->DumpRaw("aec3_capture_saturation",
aec_state_.SaturatedCapture() ? 1 : 0);
}
} // namespace
EchoRemover* EchoRemover::Create(const EchoCanceller3Config& config,
int sample_rate_hz) {
return new EchoRemoverImpl(config, sample_rate_hz);
}
} // namespace webrtc
Reference in a new issue View git blame Copy permalink