mirror of
https://github.com/mollyim/webrtc.git
synced 2025-05-17 23:57:59 +01:00
3ab308f869
This CL adds functionality for passing the information about the estimated delay to the echo remover in AEC3. The CL also adds information about how long ago the delay changed, and how long ago the delay estimate was updated. Bug: webrtc:8671 Change-Id: If274ffe0465eb550f3e186d0599c6dc6fef7f5e8 Reviewed-on: https://webrtc-review.googlesource.com/55261 Reviewed-by: Gustaf Ullberg <gustaf@webrtc.org> Commit-Queue: Per Åhgren <peah@webrtc.org> Cr-Commit-Position: refs/heads/master@{#22137}
272 lines
11 KiB
C++
272 lines
11 KiB
C++
/*
|
|
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
#include "modules/audio_processing/aec3/block_processor.h"
|
|
|
|
#include "api/optional.h"
|
|
#include "modules/audio_processing/aec3/aec3_common.h"
|
|
#include "modules/audio_processing/aec3/block_processor_metrics.h"
|
|
#include "modules/audio_processing/aec3/echo_path_variability.h"
|
|
#include "modules/audio_processing/logging/apm_data_dumper.h"
|
|
#include "rtc_base/atomicops.h"
|
|
#include "rtc_base/constructormagic.h"
|
|
#include "rtc_base/logging.h"
|
|
|
|
namespace webrtc {
|
|
namespace {
|
|
|
|
enum class BlockProcessorApiCall { kCapture, kRender };
|
|
|
|
class BlockProcessorImpl final : public BlockProcessor {
|
|
public:
|
|
BlockProcessorImpl(const EchoCanceller3Config& config,
|
|
int sample_rate_hz,
|
|
std::unique_ptr<RenderDelayBuffer> render_buffer,
|
|
std::unique_ptr<RenderDelayController> delay_controller,
|
|
std::unique_ptr<EchoRemover> echo_remover);
|
|
|
|
~BlockProcessorImpl() override;
|
|
|
|
void ProcessCapture(bool echo_path_gain_change,
|
|
bool capture_signal_saturation,
|
|
std::vector<std::vector<float>>* capture_block) override;
|
|
|
|
void BufferRender(const std::vector<std::vector<float>>& block) override;
|
|
|
|
void UpdateEchoLeakageStatus(bool leakage_detected) override;
|
|
|
|
void GetMetrics(EchoControl::Metrics* metrics) const override;
|
|
|
|
private:
|
|
static int instance_count_;
|
|
std::unique_ptr<ApmDataDumper> data_dumper_;
|
|
const EchoCanceller3Config config_;
|
|
bool capture_properly_started_ = false;
|
|
bool render_properly_started_ = false;
|
|
const size_t sample_rate_hz_;
|
|
std::unique_ptr<RenderDelayBuffer> render_buffer_;
|
|
std::unique_ptr<RenderDelayController> delay_controller_;
|
|
std::unique_ptr<EchoRemover> echo_remover_;
|
|
BlockProcessorMetrics metrics_;
|
|
RenderDelayBuffer::BufferingEvent render_event_;
|
|
size_t capture_call_counter_ = 0;
|
|
rtc::Optional<DelayEstimate> estimated_delay_;
|
|
RTC_DISALLOW_IMPLICIT_CONSTRUCTORS(BlockProcessorImpl);
|
|
};
|
|
|
|
int BlockProcessorImpl::instance_count_ = 0;
|
|
|
|
BlockProcessorImpl::BlockProcessorImpl(
|
|
const EchoCanceller3Config& config,
|
|
int sample_rate_hz,
|
|
std::unique_ptr<RenderDelayBuffer> render_buffer,
|
|
std::unique_ptr<RenderDelayController> delay_controller,
|
|
std::unique_ptr<EchoRemover> echo_remover)
|
|
: data_dumper_(
|
|
new ApmDataDumper(rtc::AtomicOps::Increment(&instance_count_))),
|
|
config_(config),
|
|
sample_rate_hz_(sample_rate_hz),
|
|
render_buffer_(std::move(render_buffer)),
|
|
delay_controller_(std::move(delay_controller)),
|
|
echo_remover_(std::move(echo_remover)),
|
|
render_event_(RenderDelayBuffer::BufferingEvent::kNone) {
|
|
RTC_DCHECK(ValidFullBandRate(sample_rate_hz_));
|
|
}
|
|
|
|
BlockProcessorImpl::~BlockProcessorImpl() = default;
|
|
|
|
void BlockProcessorImpl::ProcessCapture(
|
|
bool echo_path_gain_change,
|
|
bool capture_signal_saturation,
|
|
std::vector<std::vector<float>>* capture_block) {
|
|
RTC_DCHECK(capture_block);
|
|
RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), capture_block->size());
|
|
RTC_DCHECK_EQ(kBlockSize, (*capture_block)[0].size());
|
|
|
|
capture_call_counter_++;
|
|
|
|
data_dumper_->DumpRaw("aec3_processblock_call_order",
|
|
static_cast<int>(BlockProcessorApiCall::kCapture));
|
|
data_dumper_->DumpWav("aec3_processblock_capture_input", kBlockSize,
|
|
&(*capture_block)[0][0],
|
|
LowestBandRate(sample_rate_hz_), 1);
|
|
|
|
if (render_properly_started_) {
|
|
if (!capture_properly_started_) {
|
|
capture_properly_started_ = true;
|
|
render_buffer_->Reset();
|
|
delay_controller_->Reset();
|
|
}
|
|
} else {
|
|
// If no render data has yet arrived, do not process the capture signal.
|
|
return;
|
|
}
|
|
|
|
EchoPathVariability echo_path_variability(
|
|
echo_path_gain_change, EchoPathVariability::DelayAdjustment::kNone,
|
|
false);
|
|
|
|
if (render_event_ == RenderDelayBuffer::BufferingEvent::kRenderOverrun &&
|
|
render_properly_started_) {
|
|
echo_path_variability.delay_change =
|
|
EchoPathVariability::DelayAdjustment::kBufferFlush;
|
|
delay_controller_->Reset();
|
|
RTC_LOG(LS_WARNING) << "Reset due to render buffer overrun at block "
|
|
<< capture_call_counter_;
|
|
}
|
|
|
|
// Update the render buffers with any newly arrived render blocks and prepare
|
|
// the render buffers for reading the render data corresponding to the current
|
|
// capture block.
|
|
render_event_ = render_buffer_->PrepareCaptureProcessing();
|
|
RTC_DCHECK(RenderDelayBuffer::BufferingEvent::kRenderOverrun !=
|
|
render_event_);
|
|
if (render_event_ == RenderDelayBuffer::BufferingEvent::kRenderUnderrun) {
|
|
if (estimated_delay_ &&
|
|
estimated_delay_->quality == DelayEstimate::Quality::kRefined) {
|
|
echo_path_variability.delay_change =
|
|
EchoPathVariability::DelayAdjustment::kDelayReset;
|
|
delay_controller_->Reset();
|
|
capture_properly_started_ = false;
|
|
render_properly_started_ = false;
|
|
|
|
RTC_LOG(LS_WARNING) << "Reset due to render buffer underrrun at block "
|
|
<< capture_call_counter_;
|
|
}
|
|
} else if (render_event_ == RenderDelayBuffer::BufferingEvent::kApiCallSkew) {
|
|
// There have been too many render calls in a row. Reset to avoid noncausal
|
|
// echo.
|
|
echo_path_variability.delay_change =
|
|
EchoPathVariability::DelayAdjustment::kDelayReset;
|
|
delay_controller_->Reset();
|
|
capture_properly_started_ = false;
|
|
render_properly_started_ = false;
|
|
RTC_LOG(LS_WARNING) << "Reset due to render buffer api skew at block "
|
|
<< capture_call_counter_;
|
|
}
|
|
|
|
data_dumper_->DumpWav("aec3_processblock_capture_input2", kBlockSize,
|
|
&(*capture_block)[0][0],
|
|
LowestBandRate(sample_rate_hz_), 1);
|
|
|
|
// Compute and and apply the render delay required to achieve proper signal
|
|
// alignment.
|
|
estimated_delay_ = delay_controller_->GetDelay(
|
|
render_buffer_->GetDownsampledRenderBuffer(), (*capture_block)[0]);
|
|
|
|
if (estimated_delay_) {
|
|
if (render_buffer_->CausalDelay(estimated_delay_->delay)) {
|
|
bool delay_change = render_buffer_->SetDelay(estimated_delay_->delay);
|
|
if (delay_change) {
|
|
RTC_LOG(LS_WARNING) << "Delay changed to " << estimated_delay_->delay
|
|
<< " at block " << capture_call_counter_;
|
|
echo_path_variability.delay_change =
|
|
EchoPathVariability::DelayAdjustment::kNewDetectedDelay;
|
|
}
|
|
} else {
|
|
// A noncausal delay has been detected. This can only happen if there is
|
|
// clockdrift, an audio pipeline issue has occurred, an unreliable delay
|
|
// estimate is used or the specified minimum delay is too short.
|
|
if (estimated_delay_->quality == DelayEstimate::Quality::kRefined) {
|
|
echo_path_variability.delay_change =
|
|
EchoPathVariability::DelayAdjustment::kDelayReset;
|
|
delay_controller_->Reset();
|
|
render_buffer_->Reset();
|
|
capture_properly_started_ = false;
|
|
render_properly_started_ = false;
|
|
RTC_LOG(LS_WARNING) << "Reset due to noncausal delay at block "
|
|
<< capture_call_counter_;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Remove the echo from the capture signal.
|
|
echo_remover_->ProcessCapture(
|
|
echo_path_variability, capture_signal_saturation, estimated_delay_,
|
|
render_buffer_->GetRenderBuffer(), capture_block);
|
|
|
|
// Update the metrics.
|
|
metrics_.UpdateCapture(false);
|
|
|
|
render_event_ = RenderDelayBuffer::BufferingEvent::kNone;
|
|
}
|
|
|
|
void BlockProcessorImpl::BufferRender(
|
|
const std::vector<std::vector<float>>& block) {
|
|
RTC_DCHECK_EQ(NumBandsForRate(sample_rate_hz_), block.size());
|
|
RTC_DCHECK_EQ(kBlockSize, block[0].size());
|
|
data_dumper_->DumpRaw("aec3_processblock_call_order",
|
|
static_cast<int>(BlockProcessorApiCall::kRender));
|
|
data_dumper_->DumpWav("aec3_processblock_render_input", kBlockSize,
|
|
&block[0][0], LowestBandRate(sample_rate_hz_), 1);
|
|
data_dumper_->DumpWav("aec3_processblock_render_input2", kBlockSize,
|
|
&block[0][0], LowestBandRate(sample_rate_hz_), 1);
|
|
|
|
render_event_ = render_buffer_->Insert(block);
|
|
|
|
metrics_.UpdateRender(render_event_ !=
|
|
RenderDelayBuffer::BufferingEvent::kNone);
|
|
|
|
render_properly_started_ = true;
|
|
delay_controller_->LogRenderCall();
|
|
}
|
|
|
|
void BlockProcessorImpl::UpdateEchoLeakageStatus(bool leakage_detected) {
|
|
echo_remover_->UpdateEchoLeakageStatus(leakage_detected);
|
|
}
|
|
|
|
void BlockProcessorImpl::GetMetrics(EchoControl::Metrics* metrics) const {
|
|
echo_remover_->GetMetrics(metrics);
|
|
const int block_size_ms = sample_rate_hz_ == 8000 ? 8 : 4;
|
|
rtc::Optional<size_t> delay = render_buffer_->Delay();
|
|
metrics->delay_ms = delay ? static_cast<int>(*delay) * block_size_ms : 0;
|
|
}
|
|
|
|
} // namespace
|
|
|
|
BlockProcessor* BlockProcessor::Create(const EchoCanceller3Config& config,
|
|
int sample_rate_hz) {
|
|
std::unique_ptr<RenderDelayBuffer> render_buffer(
|
|
RenderDelayBuffer::Create(config, NumBandsForRate(sample_rate_hz)));
|
|
std::unique_ptr<RenderDelayController> delay_controller(
|
|
RenderDelayController::Create(
|
|
config, RenderDelayBuffer::DelayEstimatorOffset(config),
|
|
sample_rate_hz));
|
|
std::unique_ptr<EchoRemover> echo_remover(
|
|
EchoRemover::Create(config, sample_rate_hz));
|
|
return Create(config, sample_rate_hz, std::move(render_buffer),
|
|
std::move(delay_controller), std::move(echo_remover));
|
|
}
|
|
|
|
BlockProcessor* BlockProcessor::Create(
|
|
const EchoCanceller3Config& config,
|
|
int sample_rate_hz,
|
|
std::unique_ptr<RenderDelayBuffer> render_buffer) {
|
|
std::unique_ptr<RenderDelayController> delay_controller(
|
|
RenderDelayController::Create(
|
|
config, RenderDelayBuffer::DelayEstimatorOffset(config),
|
|
sample_rate_hz));
|
|
std::unique_ptr<EchoRemover> echo_remover(
|
|
EchoRemover::Create(config, sample_rate_hz));
|
|
return Create(config, sample_rate_hz, std::move(render_buffer),
|
|
std::move(delay_controller), std::move(echo_remover));
|
|
}
|
|
|
|
BlockProcessor* BlockProcessor::Create(
|
|
const EchoCanceller3Config& config,
|
|
int sample_rate_hz,
|
|
std::unique_ptr<RenderDelayBuffer> render_buffer,
|
|
std::unique_ptr<RenderDelayController> delay_controller,
|
|
std::unique_ptr<EchoRemover> echo_remover) {
|
|
return new BlockProcessorImpl(
|
|
config, sample_rate_hz, std::move(render_buffer),
|
|
std::move(delay_controller), std::move(echo_remover));
|
|
}
|
|
|
|
} // namespace webrtc
|