mollyim/webrtc
1
0
Fork 0
mirror of https://github.com/mollyim/webrtc.git synced 2025-05-16 23:30:48 +01:00
Code Issues Projects Releases Packages Wiki Activity Actions
webrtc/modules/audio_processing/test/audio_processing_simulator.cc
Sam Zackrisson feee1e4c36 Add flag to APM to force multichannel even with AEC3 
Currently, APM fakes multichannel in two ways:
 - With injected AECs, capture processing is only performed on the left
channel. The result is copied into the other channels.
 - With multichannel render audio, all channels are mixed into one
before analysing.

This CL adds a flag to disable these behaviors, ensuring proper
multichannel processing happens throughout the APM pipeline.

Adds killswitches to separately disable render / capture multichannel.

Additionally - AEC3 currently crashes when running with multichannel.
This CL adds the missing pieces to at least have it run without
triggering any DCHECKS, including making the high pass filter properly
handle multichannel.

Bug: webrtc:10913, webrtc:10907
Change-Id: I38795bf8f312b959fcc816a056fba2c68d4e424d
Reviewed-on: https://webrtc-review.googlesource.com/c/src/+/152483
Commit-Queue: Sam Zackrisson <saza@webrtc.org>
Reviewed-by: Per Åhgren <peah@webrtc.org>
Cr-Commit-Position: refs/heads/master@{#29248}
2019-09-20 06:36:12 +00:00

554 lines
21 KiB
C++
Raw Blame History

/*
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/audio_processing/test/audio_processing_simulator.h"
#include <algorithm>
#include <fstream>
#include <iostream>
#include <memory>
#include <string>
#include <utility>
#include <vector>
#include "api/audio/echo_canceller3_config_json.h"
#include "api/audio/echo_canceller3_factory.h"
#include "common_audio/include/audio_util.h"
#include "modules/audio_processing/aec_dump/aec_dump_factory.h"
#include "modules/audio_processing/echo_cancellation_impl.h"
#include "modules/audio_processing/echo_control_mobile_impl.h"
#include "modules/audio_processing/include/audio_processing.h"
#include "modules/audio_processing/logging/apm_data_dumper.h"
#include "modules/audio_processing/test/fake_recording_device.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/strings/json.h"
#include "rtc_base/strings/string_builder.h"
namespace webrtc {
namespace test {
namespace {
// Helper for reading JSON from a file and parsing it to an AEC3 configuration.
EchoCanceller3Config ReadAec3ConfigFromJsonFile(const std::string& filename) {
std::string json_string;
std::string s;
std::ifstream f(filename.c_str());
if (f.fail()) {
std::cout << "Failed to open the file " << filename << std::endl;
RTC_CHECK(false);
}
while (std::getline(f, s)) {
json_string += s;
}
bool parsing_successful;
EchoCanceller3Config cfg;
Aec3ConfigFromJsonString(json_string, &cfg, &parsing_successful);
if (!parsing_successful) {
std::cout << "Parsing of json string failed: " << std::endl
<< json_string << std::endl;
RTC_CHECK(false);
}
RTC_CHECK(EchoCanceller3Config::Validate(&cfg));
return cfg;
}
void CopyFromAudioFrame(const AudioFrame& src, ChannelBuffer<float>* dest) {
RTC_CHECK_EQ(src.num_channels_, dest->num_channels());
RTC_CHECK_EQ(src.samples_per_channel_, dest->num_frames());
// Copy the data from the input buffer.
std::vector<float> tmp(src.samples_per_channel_ * src.num_channels_);
S16ToFloat(src.data(), tmp.size(), tmp.data());
Deinterleave(tmp.data(), src.samples_per_channel_, src.num_channels_,
dest->channels());
}
std::string GetIndexedOutputWavFilename(const std::string& wav_name,
int counter) {
rtc::StringBuilder ss;
ss << wav_name.substr(0, wav_name.size() - 4) << "_" << counter
<< wav_name.substr(wav_name.size() - 4);
return ss.Release();
}
void WriteEchoLikelihoodGraphFileHeader(std::ofstream* output_file) {
(*output_file) << "import numpy as np" << std::endl
<< "import matplotlib.pyplot as plt" << std::endl
<< "y = np.array([";
}
void WriteEchoLikelihoodGraphFileFooter(std::ofstream* output_file) {
(*output_file) << "])" << std::endl
<< "if __name__ == '__main__':" << std::endl
<< " x = np.arange(len(y))*.01" << std::endl
<< " plt.plot(x, y)" << std::endl
<< " plt.ylabel('Echo likelihood')" << std::endl
<< " plt.xlabel('Time (s)')" << std::endl
<< " plt.show()" << std::endl;
}
// RAII class for execution time measurement. Updates the provided
// ApiCallStatistics based on the time between ScopedTimer creation and
// leaving the enclosing scope.
class ScopedTimer {
public:
ScopedTimer(ApiCallStatistics* api_call_statistics_,
ApiCallStatistics::CallType call_type)
: start_time_(rtc::TimeNanos()),
call_type_(call_type),
api_call_statistics_(api_call_statistics_) {}
~ScopedTimer() {
api_call_statistics_->Add(rtc::TimeNanos() - start_time_, call_type_);
}
private:
const int64_t start_time_;
const ApiCallStatistics::CallType call_type_;
ApiCallStatistics* const api_call_statistics_;
};
} // namespace
SimulationSettings::SimulationSettings() = default;
SimulationSettings::SimulationSettings(const SimulationSettings&) = default;
SimulationSettings::~SimulationSettings() = default;
void CopyToAudioFrame(const ChannelBuffer<float>& src, AudioFrame* dest) {
RTC_CHECK_EQ(src.num_channels(), dest->num_channels_);
RTC_CHECK_EQ(src.num_frames(), dest->samples_per_channel_);
int16_t* dest_data = dest->mutable_data();
for (size_t ch = 0; ch < dest->num_channels_; ++ch) {
for (size_t sample = 0; sample < dest->samples_per_channel_; ++sample) {
dest_data[sample * dest->num_channels_ + ch] =
src.channels()[ch][sample] * 32767;
}
}
}
AudioProcessingSimulator::AudioProcessingSimulator(
const SimulationSettings& settings,
std::unique_ptr<AudioProcessingBuilder> ap_builder)
: settings_(settings),
ap_builder_(ap_builder ? std::move(ap_builder)
: std::make_unique<AudioProcessingBuilder>()),
analog_mic_level_(settings.initial_mic_level),
fake_recording_device_(
settings.initial_mic_level,
settings_.simulate_mic_gain ? *settings.simulated_mic_kind : 0),
worker_queue_("file_writer_task_queue") {
RTC_CHECK(!settings_.dump_internal_data || WEBRTC_APM_DEBUG_DUMP == 1);
ApmDataDumper::SetActivated(settings_.dump_internal_data);
if (settings_.dump_internal_data_output_dir.has_value()) {
ApmDataDumper::SetOutputDirectory(
settings_.dump_internal_data_output_dir.value());
}
if (settings_.ed_graph_output_filename &&
!settings_.ed_graph_output_filename->empty()) {
residual_echo_likelihood_graph_writer_.open(
*settings_.ed_graph_output_filename);
RTC_CHECK(residual_echo_likelihood_graph_writer_.is_open());
WriteEchoLikelihoodGraphFileHeader(&residual_echo_likelihood_graph_writer_);
}
if (settings_.simulate_mic_gain)
RTC_LOG(LS_VERBOSE) << "Simulating analog mic gain";
}
AudioProcessingSimulator::~AudioProcessingSimulator() {
if (residual_echo_likelihood_graph_writer_.is_open()) {
WriteEchoLikelihoodGraphFileFooter(&residual_echo_likelihood_graph_writer_);
residual_echo_likelihood_graph_writer_.close();
}
}
void AudioProcessingSimulator::ProcessStream(bool fixed_interface) {
// Optionally use the fake recording device to simulate analog gain.
if (settings_.simulate_mic_gain) {
if (settings_.aec_dump_input_filename) {
// When the analog gain is simulated and an AEC dump is used as input, set
// the undo level to |aec_dump_mic_level_| to virtually restore the
// unmodified microphone signal level.
fake_recording_device_.SetUndoMicLevel(aec_dump_mic_level_);
}
if (fixed_interface) {
fake_recording_device_.SimulateAnalogGain(&fwd_frame_);
} else {
fake_recording_device_.SimulateAnalogGain(in_buf_.get());
}
// Notify the current mic level to AGC.
RTC_CHECK_EQ(AudioProcessing::kNoError,
ap_->gain_control()->set_stream_analog_level(
fake_recording_device_.MicLevel()));
} else {
// Notify the current mic level to AGC.
RTC_CHECK_EQ(AudioProcessing::kNoError,
ap_->gain_control()->set_stream_analog_level(
settings_.aec_dump_input_filename ? aec_dump_mic_level_
: analog_mic_level_));
}
// Process the current audio frame.
if (fixed_interface) {
{
const auto st = ScopedTimer(&api_call_statistics_,
ApiCallStatistics::CallType::kCapture);
RTC_CHECK_EQ(AudioProcessing::kNoError, ap_->ProcessStream(&fwd_frame_));
}
CopyFromAudioFrame(fwd_frame_, out_buf_.get());
} else {
const auto st = ScopedTimer(&api_call_statistics_,
ApiCallStatistics::CallType::kCapture);
RTC_CHECK_EQ(AudioProcessing::kNoError,
ap_->ProcessStream(in_buf_->channels(), in_config_,
out_config_, out_buf_->channels()));
}
// Store the mic level suggested by AGC.
// Note that when the analog gain is simulated and an AEC dump is used as
// input, |analog_mic_level_| will not be used with set_stream_analog_level().
analog_mic_level_ = ap_->gain_control()->stream_analog_level();
if (settings_.simulate_mic_gain) {
fake_recording_device_.SetMicLevel(analog_mic_level_);
}
if (buffer_memory_writer_) {
RTC_CHECK(!buffer_file_writer_);
buffer_memory_writer_->Write(*out_buf_);
} else if (buffer_file_writer_) {
RTC_CHECK(!buffer_memory_writer_);
buffer_file_writer_->Write(*out_buf_);
}
if (residual_echo_likelihood_graph_writer_.is_open()) {
auto stats = ap_->GetStatistics(true /*has_remote_tracks*/);
residual_echo_likelihood_graph_writer_
<< stats.residual_echo_likelihood.value_or(-1.f) << ", ";
}
++num_process_stream_calls_;
}
void AudioProcessingSimulator::ProcessReverseStream(bool fixed_interface) {
if (fixed_interface) {
{
const auto st = ScopedTimer(&api_call_statistics_,
ApiCallStatistics::CallType::kRender);
RTC_CHECK_EQ(AudioProcessing::kNoError,
ap_->ProcessReverseStream(&rev_frame_));
}
CopyFromAudioFrame(rev_frame_, reverse_out_buf_.get());
} else {
const auto st = ScopedTimer(&api_call_statistics_,
ApiCallStatistics::CallType::kRender);
RTC_CHECK_EQ(AudioProcessing::kNoError,
ap_->ProcessReverseStream(
reverse_in_buf_->channels(), reverse_in_config_,
reverse_out_config_, reverse_out_buf_->channels()));
}
if (reverse_buffer_file_writer_) {
reverse_buffer_file_writer_->Write(*reverse_out_buf_);
}
++num_reverse_process_stream_calls_;
}
void AudioProcessingSimulator::SetupBuffersConfigsOutputs(
int input_sample_rate_hz,
int output_sample_rate_hz,
int reverse_input_sample_rate_hz,
int reverse_output_sample_rate_hz,
int input_num_channels,
int output_num_channels,
int reverse_input_num_channels,
int reverse_output_num_channels) {
in_config_ = StreamConfig(input_sample_rate_hz, input_num_channels);
in_buf_.reset(new ChannelBuffer<float>(
rtc::CheckedDivExact(input_sample_rate_hz, kChunksPerSecond),
input_num_channels));
reverse_in_config_ =
StreamConfig(reverse_input_sample_rate_hz, reverse_input_num_channels);
reverse_in_buf_.reset(new ChannelBuffer<float>(
rtc::CheckedDivExact(reverse_input_sample_rate_hz, kChunksPerSecond),
reverse_input_num_channels));
out_config_ = StreamConfig(output_sample_rate_hz, output_num_channels);
out_buf_.reset(new ChannelBuffer<float>(
rtc::CheckedDivExact(output_sample_rate_hz, kChunksPerSecond),
output_num_channels));
reverse_out_config_ =
StreamConfig(reverse_output_sample_rate_hz, reverse_output_num_channels);
reverse_out_buf_.reset(new ChannelBuffer<float>(
rtc::CheckedDivExact(reverse_output_sample_rate_hz, kChunksPerSecond),
reverse_output_num_channels));
fwd_frame_.sample_rate_hz_ = input_sample_rate_hz;
fwd_frame_.samples_per_channel_ =
rtc::CheckedDivExact(fwd_frame_.sample_rate_hz_, kChunksPerSecond);
fwd_frame_.num_channels_ = input_num_channels;
rev_frame_.sample_rate_hz_ = reverse_input_sample_rate_hz;
rev_frame_.samples_per_channel_ =
rtc::CheckedDivExact(rev_frame_.sample_rate_hz_, kChunksPerSecond);
rev_frame_.num_channels_ = reverse_input_num_channels;
if (settings_.use_verbose_logging) {
rtc::LogMessage::LogToDebug(rtc::LS_VERBOSE);
std::cout << "Sample rates:" << std::endl;
std::cout << " Forward input: " << input_sample_rate_hz << std::endl;
std::cout << " Forward output: " << output_sample_rate_hz << std::endl;
std::cout << " Reverse input: " << reverse_input_sample_rate_hz
<< std::endl;
std::cout << " Reverse output: " << reverse_output_sample_rate_hz
<< std::endl;
std::cout << "Number of channels: " << std::endl;
std::cout << " Forward input: " << input_num_channels << std::endl;
std::cout << " Forward output: " << output_num_channels << std::endl;
std::cout << " Reverse input: " << reverse_input_num_channels << std::endl;
std::cout << " Reverse output: " << reverse_output_num_channels
<< std::endl;
}
SetupOutput();
}
void AudioProcessingSimulator::SetupOutput() {
if (settings_.output_filename) {
std::string filename;
if (settings_.store_intermediate_output) {
filename = GetIndexedOutputWavFilename(*settings_.output_filename,
output_reset_counter_);
} else {
filename = *settings_.output_filename;
}
std::unique_ptr<WavWriter> out_file(
new WavWriter(filename, out_config_.sample_rate_hz(),
static_cast<size_t>(out_config_.num_channels())));
buffer_file_writer_.reset(new ChannelBufferWavWriter(std::move(out_file)));
} else if (settings_.aec_dump_input_string.has_value()) {
buffer_memory_writer_ = std::make_unique<ChannelBufferVectorWriter>(
settings_.processed_capture_samples);
}
if (settings_.reverse_output_filename) {
std::string filename;
if (settings_.store_intermediate_output) {
filename = GetIndexedOutputWavFilename(*settings_.reverse_output_filename,
output_reset_counter_);
} else {
filename = *settings_.reverse_output_filename;
}
std::unique_ptr<WavWriter> reverse_out_file(
new WavWriter(filename, reverse_out_config_.sample_rate_hz(),
static_cast<size_t>(reverse_out_config_.num_channels())));
reverse_buffer_file_writer_.reset(
new ChannelBufferWavWriter(std::move(reverse_out_file)));
}
++output_reset_counter_;
}
void AudioProcessingSimulator::DestroyAudioProcessor() {
if (settings_.aec_dump_output_filename) {
ap_->DetachAecDump();
}
}
void AudioProcessingSimulator::CreateAudioProcessor() {
Config config;
AudioProcessing::Config apm_config;
std::unique_ptr<EchoControlFactory> echo_control_factory;
if (settings_.use_ts) {
config.Set<ExperimentalNs>(new ExperimentalNs(*settings_.use_ts));
}
if (settings_.experimental_multi_channel) {
apm_config.pipeline.experimental_multi_channel =
*settings_.experimental_multi_channel;
}
if (settings_.use_agc2) {
apm_config.gain_controller2.enabled = *settings_.use_agc2;
if (settings_.agc2_fixed_gain_db) {
apm_config.gain_controller2.fixed_digital.gain_db =
*settings_.agc2_fixed_gain_db;
}
if (settings_.agc2_use_adaptive_gain) {
apm_config.gain_controller2.adaptive_digital.enabled =
*settings_.agc2_use_adaptive_gain;
apm_config.gain_controller2.adaptive_digital.level_estimator =
settings_.agc2_adaptive_level_estimator;
}
}
if (settings_.use_pre_amplifier) {
apm_config.pre_amplifier.enabled = *settings_.use_pre_amplifier;
if (settings_.pre_amplifier_gain_factor) {
apm_config.pre_amplifier.fixed_gain_factor =
*settings_.pre_amplifier_gain_factor;
}
}
const bool use_legacy_aec = settings_.use_aec && *settings_.use_aec &&
settings_.use_legacy_aec &&
*settings_.use_legacy_aec;
const bool use_aec = settings_.use_aec && *settings_.use_aec;
const bool use_aecm = settings_.use_aecm && *settings_.use_aecm;
if (use_legacy_aec || use_aec || use_aecm) {
apm_config.echo_canceller.enabled = true;
apm_config.echo_canceller.mobile_mode = use_aecm;
apm_config.echo_canceller.use_legacy_aec = use_legacy_aec;
}
RTC_CHECK(!(use_legacy_aec && settings_.aec_settings_filename))
<< "The legacy AEC cannot be configured using settings";
if (use_aec && !use_legacy_aec) {
EchoCanceller3Config cfg;
if (settings_.aec_settings_filename) {
if (settings_.use_verbose_logging) {
std::cout << "Reading AEC Parameters from JSON input." << std::endl;
}
cfg = ReadAec3ConfigFromJsonFile(*settings_.aec_settings_filename);
echo_control_factory.reset(new EchoCanceller3Factory(cfg));
}
if (settings_.print_aec_parameter_values) {
if (!settings_.use_quiet_output) {
std::cout << "AEC settings:" << std::endl;
}
std::cout << Aec3ConfigToJsonString(cfg) << std::endl;
}
}
if (settings_.use_drift_compensation && *settings_.use_drift_compensation) {
RTC_LOG(LS_ERROR) << "Ignoring deprecated setting: AEC2 drift compensation";
}
if (settings_.aec_suppression_level) {
auto level = static_cast<webrtc::EchoCancellationImpl::SuppressionLevel>(
*settings_.aec_suppression_level);
if (level ==
webrtc::EchoCancellationImpl::SuppressionLevel::kLowSuppression) {
RTC_LOG(LS_ERROR) << "Ignoring deprecated setting: AEC2 low suppression";
} else {
apm_config.echo_canceller.legacy_moderate_suppression_level =
(level == webrtc::EchoCancellationImpl::SuppressionLevel::
kModerateSuppression);
}
}
if (settings_.use_hpf) {
apm_config.high_pass_filter.enabled = *settings_.use_hpf;
}
if (settings_.use_refined_adaptive_filter) {
config.Set<RefinedAdaptiveFilter>(
new RefinedAdaptiveFilter(*settings_.use_refined_adaptive_filter));
}
config.Set<ExtendedFilter>(new ExtendedFilter(
!settings_.use_extended_filter || *settings_.use_extended_filter));
config.Set<DelayAgnostic>(new DelayAgnostic(!settings_.use_delay_agnostic ||
*settings_.use_delay_agnostic));
config.Set<ExperimentalAgc>(new ExperimentalAgc(
!settings_.use_experimental_agc || *settings_.use_experimental_agc,
!!settings_.use_experimental_agc_agc2_level_estimator &&
*settings_.use_experimental_agc_agc2_level_estimator,
!!settings_.experimental_agc_disable_digital_adaptive &&
*settings_.experimental_agc_disable_digital_adaptive,
!!settings_.experimental_agc_analyze_before_aec &&
*settings_.experimental_agc_analyze_before_aec));
if (settings_.use_ed) {
apm_config.residual_echo_detector.enabled = *settings_.use_ed;
}
if (settings_.maximum_internal_processing_rate) {
apm_config.pipeline.maximum_internal_processing_rate =
*settings_.maximum_internal_processing_rate;
}
RTC_CHECK(ap_builder_);
if (echo_control_factory) {
ap_builder_->SetEchoControlFactory(std::move(echo_control_factory));
}
ap_.reset((*ap_builder_).Create(config));
RTC_CHECK(ap_);
ap_->ApplyConfig(apm_config);
if (settings_.use_agc) {
RTC_CHECK_EQ(AudioProcessing::kNoError,
ap_->gain_control()->Enable(*settings_.use_agc));
}
if (settings_.use_ns) {
RTC_CHECK_EQ(AudioProcessing::kNoError,
ap_->noise_suppression()->Enable(*settings_.use_ns));
}
if (settings_.use_le) {
RTC_CHECK_EQ(AudioProcessing::kNoError,
ap_->level_estimator()->Enable(*settings_.use_le));
}
if (settings_.use_vad) {
RTC_CHECK_EQ(AudioProcessing::kNoError,
ap_->voice_detection()->Enable(*settings_.use_vad));
}
if (settings_.use_agc_limiter) {
RTC_CHECK_EQ(AudioProcessing::kNoError, ap_->gain_control()->enable_limiter(
*settings_.use_agc_limiter));
}
if (settings_.agc_target_level) {
RTC_CHECK_EQ(AudioProcessing::kNoError,
ap_->gain_control()->set_target_level_dbfs(
*settings_.agc_target_level));
}
if (settings_.agc_compression_gain) {
RTC_CHECK_EQ(AudioProcessing::kNoError,
ap_->gain_control()->set_compression_gain_db(
*settings_.agc_compression_gain));
}
if (settings_.agc_mode) {
RTC_CHECK_EQ(
AudioProcessing::kNoError,
ap_->gain_control()->set_mode(
static_cast<webrtc::GainControl::Mode>(*settings_.agc_mode)));
}
if (settings_.vad_likelihood) {
RTC_CHECK_EQ(AudioProcessing::kNoError,
ap_->voice_detection()->set_likelihood(
static_cast<webrtc::VoiceDetection::Likelihood>(
*settings_.vad_likelihood)));
}
if (settings_.ns_level) {
RTC_CHECK_EQ(
AudioProcessing::kNoError,
ap_->noise_suppression()->set_level(
static_cast<NoiseSuppression::Level>(*settings_.ns_level)));
}
if (settings_.use_ts) {
ap_->set_stream_key_pressed(*settings_.use_ts);
}
if (settings_.aec_dump_output_filename) {
ap_->AttachAecDump(AecDumpFactory::Create(
*settings_.aec_dump_output_filename, -1, &worker_queue_));
}
}
} // namespace test
} // namespace webrtc
Reference in a new issue View git blame Copy permalink