b5c48b97f6
Ongoing fixes and improvements, transient suppressor is gone. Also, dropping isac because it doesn't seem to be useful, and is just build system deadweight now. Upstream references: Version: 131.0.6778.200 WebRTC: 79aff54b0fa9238ce3518dd9eaf9610cd6f22e82 Chromium: 2a19506ad24af755f2a215a4c61f775393e0db42
466 lines
18 KiB
C++
466 lines
18 KiB
C++
/*
|
|
* Copyright (c) 2017 The WebRTC project authors. All Rights Reserved.
|
|
*
|
|
* Use of this source code is governed by a BSD-style license
|
|
* that can be found in the LICENSE file in the root of the source
|
|
* tree. An additional intellectual property rights grant can be found
|
|
* in the file PATENTS. All contributing project authors may
|
|
* be found in the AUTHORS file in the root of the source tree.
|
|
*/
|
|
|
|
#include "modules/audio_processing/aec3/suppression_gain.h"
|
|
|
|
#include <math.h>
|
|
#include <stddef.h>
|
|
|
|
#include <algorithm>
|
|
#include <numeric>
|
|
|
|
#include "modules/audio_processing/aec3/dominant_nearend_detector.h"
|
|
#include "modules/audio_processing/aec3/moving_average.h"
|
|
#include "modules/audio_processing/aec3/subband_nearend_detector.h"
|
|
#include "modules/audio_processing/aec3/vector_math.h"
|
|
#include "modules/audio_processing/logging/apm_data_dumper.h"
|
|
#include "rtc_base/checks.h"
|
|
#include "system_wrappers/include/field_trial.h"
|
|
|
|
namespace webrtc {
|
|
namespace {
|
|
|
|
void LimitLowFrequencyGains(std::array<float, kFftLengthBy2Plus1>* gain) {
|
|
// Limit the low frequency gains to avoid the impact of the high-pass filter
|
|
// on the lower-frequency gain influencing the overall achieved gain.
|
|
(*gain)[0] = (*gain)[1] = std::min((*gain)[1], (*gain)[2]);
|
|
}
|
|
|
|
void LimitHighFrequencyGains(bool conservative_hf_suppression,
|
|
std::array<float, kFftLengthBy2Plus1>* gain) {
|
|
// Limit the high frequency gains to avoid echo leakage due to an imperfect
|
|
// filter.
|
|
constexpr size_t kFirstBandToLimit = (64 * 2000) / 8000;
|
|
const float min_upper_gain = (*gain)[kFirstBandToLimit];
|
|
std::for_each(
|
|
gain->begin() + kFirstBandToLimit + 1, gain->end(),
|
|
[min_upper_gain](float& a) { a = std::min(a, min_upper_gain); });
|
|
(*gain)[kFftLengthBy2] = (*gain)[kFftLengthBy2Minus1];
|
|
|
|
if (conservative_hf_suppression) {
|
|
// Limits the gain in the frequencies for which the adaptive filter has not
|
|
// converged.
|
|
// TODO(peah): Make adaptive to take the actual filter error into account.
|
|
constexpr size_t kUpperAccurateBandPlus1 = 29;
|
|
|
|
constexpr float oneByBandsInSum =
|
|
1 / static_cast<float>(kUpperAccurateBandPlus1 - 20);
|
|
const float hf_gain_bound =
|
|
std::accumulate(gain->begin() + 20,
|
|
gain->begin() + kUpperAccurateBandPlus1, 0.f) *
|
|
oneByBandsInSum;
|
|
|
|
std::for_each(
|
|
gain->begin() + kUpperAccurateBandPlus1, gain->end(),
|
|
[hf_gain_bound](float& a) { a = std::min(a, hf_gain_bound); });
|
|
}
|
|
}
|
|
|
|
// Scales the echo according to assessed audibility at the other end.
|
|
void WeightEchoForAudibility(const EchoCanceller3Config& config,
|
|
rtc::ArrayView<const float> echo,
|
|
rtc::ArrayView<float> weighted_echo) {
|
|
RTC_DCHECK_EQ(kFftLengthBy2Plus1, echo.size());
|
|
RTC_DCHECK_EQ(kFftLengthBy2Plus1, weighted_echo.size());
|
|
|
|
auto weigh = [](float threshold, float normalizer, size_t begin, size_t end,
|
|
rtc::ArrayView<const float> echo,
|
|
rtc::ArrayView<float> weighted_echo) {
|
|
for (size_t k = begin; k < end; ++k) {
|
|
if (echo[k] < threshold) {
|
|
float tmp = (threshold - echo[k]) * normalizer;
|
|
weighted_echo[k] = echo[k] * std::max(0.f, 1.f - tmp * tmp);
|
|
} else {
|
|
weighted_echo[k] = echo[k];
|
|
}
|
|
}
|
|
};
|
|
|
|
float threshold = config.echo_audibility.floor_power *
|
|
config.echo_audibility.audibility_threshold_lf;
|
|
float normalizer = 1.f / (threshold - config.echo_audibility.floor_power);
|
|
weigh(threshold, normalizer, 0, 3, echo, weighted_echo);
|
|
|
|
threshold = config.echo_audibility.floor_power *
|
|
config.echo_audibility.audibility_threshold_mf;
|
|
normalizer = 1.f / (threshold - config.echo_audibility.floor_power);
|
|
weigh(threshold, normalizer, 3, 7, echo, weighted_echo);
|
|
|
|
threshold = config.echo_audibility.floor_power *
|
|
config.echo_audibility.audibility_threshold_hf;
|
|
normalizer = 1.f / (threshold - config.echo_audibility.floor_power);
|
|
weigh(threshold, normalizer, 7, kFftLengthBy2Plus1, echo, weighted_echo);
|
|
}
|
|
|
|
} // namespace
|
|
|
|
std::atomic<int> SuppressionGain::instance_count_(0);
|
|
|
|
float SuppressionGain::UpperBandsGain(
|
|
rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>> echo_spectrum,
|
|
rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
|
|
comfort_noise_spectrum,
|
|
const std::optional<int>& narrow_peak_band,
|
|
bool saturated_echo,
|
|
const Block& render,
|
|
const std::array<float, kFftLengthBy2Plus1>& low_band_gain) const {
|
|
RTC_DCHECK_LT(0, render.NumBands());
|
|
if (render.NumBands() == 1) {
|
|
return 1.f;
|
|
}
|
|
const int num_render_channels = render.NumChannels();
|
|
|
|
if (narrow_peak_band &&
|
|
(*narrow_peak_band > static_cast<int>(kFftLengthBy2Plus1 - 10))) {
|
|
return 0.001f;
|
|
}
|
|
|
|
constexpr size_t kLowBandGainLimit = kFftLengthBy2 / 2;
|
|
const float gain_below_8_khz = *std::min_element(
|
|
low_band_gain.begin() + kLowBandGainLimit, low_band_gain.end());
|
|
|
|
// Always attenuate the upper bands when there is saturated echo.
|
|
if (saturated_echo) {
|
|
return std::min(0.001f, gain_below_8_khz);
|
|
}
|
|
|
|
// Compute the upper and lower band energies.
|
|
const auto sum_of_squares = [](float a, float b) { return a + b * b; };
|
|
float low_band_energy = 0.f;
|
|
for (int ch = 0; ch < num_render_channels; ++ch) {
|
|
const float channel_energy =
|
|
std::accumulate(render.begin(/*band=*/0, ch),
|
|
render.end(/*band=*/0, ch), 0.0f, sum_of_squares);
|
|
low_band_energy = std::max(low_band_energy, channel_energy);
|
|
}
|
|
float high_band_energy = 0.f;
|
|
for (int k = 1; k < render.NumBands(); ++k) {
|
|
for (int ch = 0; ch < num_render_channels; ++ch) {
|
|
const float energy = std::accumulate(
|
|
render.begin(k, ch), render.end(k, ch), 0.f, sum_of_squares);
|
|
high_band_energy = std::max(high_band_energy, energy);
|
|
}
|
|
}
|
|
|
|
// If there is more power in the lower frequencies than the upper frequencies,
|
|
// or if the power in upper frequencies is low, do not bound the gain in the
|
|
// upper bands.
|
|
float anti_howling_gain;
|
|
const float activation_threshold =
|
|
kBlockSize * config_.suppressor.high_bands_suppression
|
|
.anti_howling_activation_threshold;
|
|
if (high_band_energy < std::max(low_band_energy, activation_threshold)) {
|
|
anti_howling_gain = 1.f;
|
|
} else {
|
|
// In all other cases, bound the gain for upper frequencies.
|
|
RTC_DCHECK_LE(low_band_energy, high_band_energy);
|
|
RTC_DCHECK_NE(0.f, high_band_energy);
|
|
anti_howling_gain =
|
|
config_.suppressor.high_bands_suppression.anti_howling_gain *
|
|
sqrtf(low_band_energy / high_band_energy);
|
|
}
|
|
|
|
float gain_bound = 1.f;
|
|
if (!dominant_nearend_detector_->IsNearendState()) {
|
|
// Bound the upper gain during significant echo activity.
|
|
const auto& cfg = config_.suppressor.high_bands_suppression;
|
|
auto low_frequency_energy = [](rtc::ArrayView<const float> spectrum) {
|
|
RTC_DCHECK_LE(16, spectrum.size());
|
|
return std::accumulate(spectrum.begin() + 1, spectrum.begin() + 16, 0.f);
|
|
};
|
|
for (size_t ch = 0; ch < num_capture_channels_; ++ch) {
|
|
const float echo_sum = low_frequency_energy(echo_spectrum[ch]);
|
|
const float noise_sum = low_frequency_energy(comfort_noise_spectrum[ch]);
|
|
if (echo_sum > cfg.enr_threshold * noise_sum) {
|
|
gain_bound = cfg.max_gain_during_echo;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
|
|
// Choose the gain as the minimum of the lower and upper gains.
|
|
return std::min(std::min(gain_below_8_khz, anti_howling_gain), gain_bound);
|
|
}
|
|
|
|
// Computes the gain to reduce the echo to a non audible level.
|
|
void SuppressionGain::GainToNoAudibleEcho(
|
|
const std::array<float, kFftLengthBy2Plus1>& nearend,
|
|
const std::array<float, kFftLengthBy2Plus1>& echo,
|
|
const std::array<float, kFftLengthBy2Plus1>& masker,
|
|
std::array<float, kFftLengthBy2Plus1>* gain) const {
|
|
const auto& p = dominant_nearend_detector_->IsNearendState() ? nearend_params_
|
|
: normal_params_;
|
|
for (size_t k = 0; k < gain->size(); ++k) {
|
|
float enr = echo[k] / (nearend[k] + 1.f); // Echo-to-nearend ratio.
|
|
float emr = echo[k] / (masker[k] + 1.f); // Echo-to-masker (noise) ratio.
|
|
float g = 1.0f;
|
|
if (enr > p.enr_transparent_[k] && emr > p.emr_transparent_[k]) {
|
|
g = (p.enr_suppress_[k] - enr) /
|
|
(p.enr_suppress_[k] - p.enr_transparent_[k]);
|
|
g = std::max(g, p.emr_transparent_[k] / emr);
|
|
}
|
|
(*gain)[k] = g;
|
|
}
|
|
}
|
|
|
|
// Compute the minimum gain as the attenuating gain to put the signal just
|
|
// above the zero sample values.
|
|
void SuppressionGain::GetMinGain(
|
|
rtc::ArrayView<const float> weighted_residual_echo,
|
|
rtc::ArrayView<const float> last_nearend,
|
|
rtc::ArrayView<const float> last_echo,
|
|
bool low_noise_render,
|
|
bool saturated_echo,
|
|
rtc::ArrayView<float> min_gain) const {
|
|
if (!saturated_echo) {
|
|
const float min_echo_power =
|
|
low_noise_render ? config_.echo_audibility.low_render_limit
|
|
: config_.echo_audibility.normal_render_limit;
|
|
|
|
for (size_t k = 0; k < min_gain.size(); ++k) {
|
|
min_gain[k] = weighted_residual_echo[k] > 0.f
|
|
? min_echo_power / weighted_residual_echo[k]
|
|
: 1.f;
|
|
min_gain[k] = std::min(min_gain[k], 1.f);
|
|
}
|
|
|
|
if (!initial_state_ ||
|
|
config_.suppressor.lf_smoothing_during_initial_phase) {
|
|
const float& dec = dominant_nearend_detector_->IsNearendState()
|
|
? nearend_params_.max_dec_factor_lf
|
|
: normal_params_.max_dec_factor_lf;
|
|
|
|
for (int k = 0; k <= config_.suppressor.last_lf_smoothing_band; ++k) {
|
|
// Make sure the gains of the low frequencies do not decrease too
|
|
// quickly after strong nearend.
|
|
if (last_nearend[k] > last_echo[k] ||
|
|
k <= config_.suppressor.last_permanent_lf_smoothing_band) {
|
|
min_gain[k] = std::max(min_gain[k], last_gain_[k] * dec);
|
|
min_gain[k] = std::min(min_gain[k], 1.f);
|
|
}
|
|
}
|
|
}
|
|
} else {
|
|
std::fill(min_gain.begin(), min_gain.end(), 0.f);
|
|
}
|
|
}
|
|
|
|
// Compute the maximum gain by limiting the gain increase from the previous
|
|
// gain.
|
|
void SuppressionGain::GetMaxGain(rtc::ArrayView<float> max_gain) const {
|
|
const auto& inc = dominant_nearend_detector_->IsNearendState()
|
|
? nearend_params_.max_inc_factor
|
|
: normal_params_.max_inc_factor;
|
|
const auto& floor = config_.suppressor.floor_first_increase;
|
|
for (size_t k = 0; k < max_gain.size(); ++k) {
|
|
max_gain[k] = std::min(std::max(last_gain_[k] * inc, floor), 1.f);
|
|
}
|
|
}
|
|
|
|
void SuppressionGain::LowerBandGain(
|
|
bool low_noise_render,
|
|
const AecState& aec_state,
|
|
rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
|
|
suppressor_input,
|
|
rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>> residual_echo,
|
|
rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>> comfort_noise,
|
|
bool clock_drift,
|
|
std::array<float, kFftLengthBy2Plus1>* gain) {
|
|
gain->fill(1.f);
|
|
const bool saturated_echo = aec_state.SaturatedEcho();
|
|
std::array<float, kFftLengthBy2Plus1> max_gain;
|
|
GetMaxGain(max_gain);
|
|
|
|
for (size_t ch = 0; ch < num_capture_channels_; ++ch) {
|
|
std::array<float, kFftLengthBy2Plus1> G;
|
|
std::array<float, kFftLengthBy2Plus1> nearend;
|
|
nearend_smoothers_[ch].Average(suppressor_input[ch], nearend);
|
|
|
|
// Weight echo power in terms of audibility.
|
|
std::array<float, kFftLengthBy2Plus1> weighted_residual_echo;
|
|
WeightEchoForAudibility(config_, residual_echo[ch], weighted_residual_echo);
|
|
|
|
std::array<float, kFftLengthBy2Plus1> min_gain;
|
|
GetMinGain(weighted_residual_echo, last_nearend_[ch], last_echo_[ch],
|
|
low_noise_render, saturated_echo, min_gain);
|
|
|
|
GainToNoAudibleEcho(nearend, weighted_residual_echo, comfort_noise[0], &G);
|
|
|
|
// Clamp gains.
|
|
for (size_t k = 0; k < gain->size(); ++k) {
|
|
G[k] = std::max(std::min(G[k], max_gain[k]), min_gain[k]);
|
|
(*gain)[k] = std::min((*gain)[k], G[k]);
|
|
}
|
|
|
|
// Store data required for the gain computation of the next block.
|
|
std::copy(nearend.begin(), nearend.end(), last_nearend_[ch].begin());
|
|
std::copy(weighted_residual_echo.begin(), weighted_residual_echo.end(),
|
|
last_echo_[ch].begin());
|
|
}
|
|
|
|
LimitLowFrequencyGains(gain);
|
|
// Use conservative high-frequency gains during clock-drift or when not in
|
|
// dominant nearend.
|
|
if (!dominant_nearend_detector_->IsNearendState() || clock_drift ||
|
|
config_.suppressor.conservative_hf_suppression) {
|
|
LimitHighFrequencyGains(config_.suppressor.conservative_hf_suppression,
|
|
gain);
|
|
}
|
|
|
|
// Store computed gains.
|
|
std::copy(gain->begin(), gain->end(), last_gain_.begin());
|
|
|
|
// Transform gains to amplitude domain.
|
|
aec3::VectorMath(optimization_).Sqrt(*gain);
|
|
}
|
|
|
|
SuppressionGain::SuppressionGain(const EchoCanceller3Config& config,
|
|
Aec3Optimization optimization,
|
|
int sample_rate_hz,
|
|
size_t num_capture_channels)
|
|
: data_dumper_(new ApmDataDumper(instance_count_.fetch_add(1) + 1)),
|
|
optimization_(optimization),
|
|
config_(config),
|
|
num_capture_channels_(num_capture_channels),
|
|
state_change_duration_blocks_(
|
|
static_cast<int>(config_.filter.config_change_duration_blocks)),
|
|
last_nearend_(num_capture_channels_, {0}),
|
|
last_echo_(num_capture_channels_, {0}),
|
|
nearend_smoothers_(
|
|
num_capture_channels_,
|
|
aec3::MovingAverage(kFftLengthBy2Plus1,
|
|
config.suppressor.nearend_average_blocks)),
|
|
nearend_params_(config_.suppressor.last_lf_band,
|
|
config_.suppressor.first_hf_band,
|
|
config_.suppressor.nearend_tuning),
|
|
normal_params_(config_.suppressor.last_lf_band,
|
|
config_.suppressor.first_hf_band,
|
|
config_.suppressor.normal_tuning),
|
|
use_unbounded_echo_spectrum_(config.suppressor.dominant_nearend_detection
|
|
.use_unbounded_echo_spectrum) {
|
|
RTC_DCHECK_LT(0, state_change_duration_blocks_);
|
|
last_gain_.fill(1.f);
|
|
if (config_.suppressor.use_subband_nearend_detection) {
|
|
dominant_nearend_detector_ = std::make_unique<SubbandNearendDetector>(
|
|
config_.suppressor.subband_nearend_detection, num_capture_channels_);
|
|
} else {
|
|
dominant_nearend_detector_ = std::make_unique<DominantNearendDetector>(
|
|
config_.suppressor.dominant_nearend_detection, num_capture_channels_);
|
|
}
|
|
RTC_DCHECK(dominant_nearend_detector_);
|
|
}
|
|
|
|
SuppressionGain::~SuppressionGain() = default;
|
|
|
|
void SuppressionGain::GetGain(
|
|
rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
|
|
nearend_spectrum,
|
|
rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>> echo_spectrum,
|
|
rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
|
|
residual_echo_spectrum,
|
|
rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
|
|
residual_echo_spectrum_unbounded,
|
|
rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
|
|
comfort_noise_spectrum,
|
|
const RenderSignalAnalyzer& render_signal_analyzer,
|
|
const AecState& aec_state,
|
|
const Block& render,
|
|
bool clock_drift,
|
|
float* high_bands_gain,
|
|
std::array<float, kFftLengthBy2Plus1>* low_band_gain) {
|
|
RTC_DCHECK(high_bands_gain);
|
|
RTC_DCHECK(low_band_gain);
|
|
|
|
// Choose residual echo spectrum for dominant nearend detection.
|
|
const auto echo = use_unbounded_echo_spectrum_
|
|
? residual_echo_spectrum_unbounded
|
|
: residual_echo_spectrum;
|
|
|
|
// Update the nearend state selection.
|
|
dominant_nearend_detector_->Update(nearend_spectrum, echo,
|
|
comfort_noise_spectrum, initial_state_);
|
|
|
|
// Compute gain for the lower band.
|
|
bool low_noise_render = low_render_detector_.Detect(render);
|
|
LowerBandGain(low_noise_render, aec_state, nearend_spectrum,
|
|
residual_echo_spectrum, comfort_noise_spectrum, clock_drift,
|
|
low_band_gain);
|
|
|
|
// Compute the gain for the upper bands.
|
|
const std::optional<int> narrow_peak_band =
|
|
render_signal_analyzer.NarrowPeakBand();
|
|
|
|
*high_bands_gain =
|
|
UpperBandsGain(echo_spectrum, comfort_noise_spectrum, narrow_peak_band,
|
|
aec_state.SaturatedEcho(), render, *low_band_gain);
|
|
|
|
data_dumper_->DumpRaw("aec3_dominant_nearend",
|
|
dominant_nearend_detector_->IsNearendState());
|
|
}
|
|
|
|
void SuppressionGain::SetInitialState(bool state) {
|
|
initial_state_ = state;
|
|
if (state) {
|
|
initial_state_change_counter_ = state_change_duration_blocks_;
|
|
} else {
|
|
initial_state_change_counter_ = 0;
|
|
}
|
|
}
|
|
|
|
// Detects when the render signal can be considered to have low power and
|
|
// consist of stationary noise.
|
|
bool SuppressionGain::LowNoiseRenderDetector::Detect(const Block& render) {
|
|
float x2_sum = 0.f;
|
|
float x2_max = 0.f;
|
|
for (int ch = 0; ch < render.NumChannels(); ++ch) {
|
|
for (float x_k : render.View(/*band=*/0, ch)) {
|
|
const float x2 = x_k * x_k;
|
|
x2_sum += x2;
|
|
x2_max = std::max(x2_max, x2);
|
|
}
|
|
}
|
|
x2_sum = x2_sum / render.NumChannels();
|
|
|
|
constexpr float kThreshold = 50.f * 50.f * 64.f;
|
|
const bool low_noise_render =
|
|
average_power_ < kThreshold && x2_max < 3 * average_power_;
|
|
average_power_ = average_power_ * 0.9f + x2_sum * 0.1f;
|
|
return low_noise_render;
|
|
}
|
|
|
|
SuppressionGain::GainParameters::GainParameters(
|
|
int last_lf_band,
|
|
int first_hf_band,
|
|
const EchoCanceller3Config::Suppressor::Tuning& tuning)
|
|
: max_inc_factor(tuning.max_inc_factor),
|
|
max_dec_factor_lf(tuning.max_dec_factor_lf) {
|
|
// Compute per-band masking thresholds.
|
|
RTC_DCHECK_LT(last_lf_band, first_hf_band);
|
|
auto& lf = tuning.mask_lf;
|
|
auto& hf = tuning.mask_hf;
|
|
RTC_DCHECK_LT(lf.enr_transparent, lf.enr_suppress);
|
|
RTC_DCHECK_LT(hf.enr_transparent, hf.enr_suppress);
|
|
for (int k = 0; k < static_cast<int>(kFftLengthBy2Plus1); k++) {
|
|
float a;
|
|
if (k <= last_lf_band) {
|
|
a = 0.f;
|
|
} else if (k < first_hf_band) {
|
|
a = (k - last_lf_band) / static_cast<float>(first_hf_band - last_lf_band);
|
|
} else {
|
|
a = 1.f;
|
|
}
|
|
enr_transparent_[k] = (1 - a) * lf.enr_transparent + a * hf.enr_transparent;
|
|
enr_suppress_[k] = (1 - a) * lf.enr_suppress + a * hf.enr_suppress;
|
|
emr_transparent_[k] = (1 - a) * lf.emr_transparent + a * hf.emr_transparent;
|
|
}
|
|
}
|
|
|
|
} // namespace webrtc
|