AnotherFoxGuy/webrtc-audio-processing
1
0
Fork 0
Code Pull Requests Activity
webrtc-audio-processing/webrtc/modules/audio_processing/agc2/input_volume_controller.cc
Arun Raghavan b5c48b97f6 Bump to WebRTC M131 release 
Ongoing fixes and improvements, transient suppressor is gone. Also,
dropping isac because it doesn't seem to be useful, and is just build
system deadweight now.

Upstream references:

  Version: 131.0.6778.200
  WebRTC: 79aff54b0fa9238ce3518dd9eaf9610cd6f22e82
  Chromium: 2a19506ad24af755f2a215a4c61f775393e0db42
2024-12-26 12:55:16 -05:00

581 lines
21 KiB
C++
Raw Permalink Blame History

/*
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/audio_processing/agc2/input_volume_controller.h"
#include <algorithm>
#include <cmath>
#include "api/array_view.h"
#include "modules/audio_processing/agc2/gain_map_internal.h"
#include "modules/audio_processing/agc2/input_volume_stats_reporter.h"
#include "modules/audio_processing/include/audio_frame_view.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"
#include "rtc_base/numerics/safe_minmax.h"
#include "system_wrappers/include/field_trial.h"
#include "system_wrappers/include/metrics.h"
namespace webrtc {
namespace {
// Amount of error we tolerate in the microphone input volume (presumably due to
// OS quantization) before we assume the user has manually adjusted the volume.
constexpr int kVolumeQuantizationSlack = 25;
constexpr int kMaxInputVolume = 255;
static_assert(kGainMapSize > kMaxInputVolume, "gain map too small");
// Maximum absolute RMS error.
constexpr int KMaxAbsRmsErrorDbfs = 15;
static_assert(KMaxAbsRmsErrorDbfs > 0, "");
using Agc1ClippingPredictorConfig = AudioProcessing::Config::GainController1::
AnalogGainController::ClippingPredictor;
// TODO(webrtc:7494): Hardcode clipping predictor parameters and remove this
// function after no longer needed in the ctor.
Agc1ClippingPredictorConfig CreateClippingPredictorConfig(bool enabled) {
Agc1ClippingPredictorConfig config;
config.enabled = enabled;
return config;
}
// Returns an input volume in the [`min_input_volume`, `kMaxInputVolume`] range
// that reduces `gain_error_db`, which is a gain error estimated when
// `input_volume` was applied, according to a fixed gain map.
int ComputeVolumeUpdate(int gain_error_db,
int input_volume,
int min_input_volume) {
RTC_DCHECK_GE(input_volume, 0);
RTC_DCHECK_LE(input_volume, kMaxInputVolume);
if (gain_error_db == 0) {
return input_volume;
}
int new_volume = input_volume;
if (gain_error_db > 0) {
while (kGainMap[new_volume] - kGainMap[input_volume] < gain_error_db &&
new_volume < kMaxInputVolume) {
++new_volume;
}
} else {
while (kGainMap[new_volume] - kGainMap[input_volume] > gain_error_db &&
new_volume > min_input_volume) {
--new_volume;
}
}
return new_volume;
}
// Returns the proportion of samples in the buffer which are at full-scale
// (and presumably clipped).
float ComputeClippedRatio(const float* const* audio,
size_t num_channels,
size_t samples_per_channel) {
RTC_DCHECK_GT(samples_per_channel, 0);
int num_clipped = 0;
for (size_t ch = 0; ch < num_channels; ++ch) {
int num_clipped_in_ch = 0;
for (size_t i = 0; i < samples_per_channel; ++i) {
RTC_DCHECK(audio[ch]);
if (audio[ch][i] >= 32767.0f || audio[ch][i] <= -32768.0f) {
++num_clipped_in_ch;
}
}
num_clipped = std::max(num_clipped, num_clipped_in_ch);
}
return static_cast<float>(num_clipped) / (samples_per_channel);
}
void LogClippingMetrics(int clipping_rate) {
RTC_LOG(LS_INFO) << "[AGC2] Input clipping rate: " << clipping_rate << "%";
RTC_HISTOGRAM_COUNTS_LINEAR(/*name=*/"WebRTC.Audio.Agc.InputClippingRate",
/*sample=*/clipping_rate, /*min=*/0, /*max=*/100,
/*bucket_count=*/50);
}
// Compares `speech_level_dbfs` to the [`target_range_min_dbfs`,
// `target_range_max_dbfs`] range and returns the error to be compensated via
// input volume adjustment. Returns a positive value when the level is below
// the range, a negative value when the level is above the range, zero
// otherwise.
int GetSpeechLevelRmsErrorDb(float speech_level_dbfs,
int target_range_min_dbfs,
int target_range_max_dbfs) {
constexpr float kMinSpeechLevelDbfs = -90.0f;
constexpr float kMaxSpeechLevelDbfs = 30.0f;
RTC_DCHECK_GE(speech_level_dbfs, kMinSpeechLevelDbfs);
RTC_DCHECK_LE(speech_level_dbfs, kMaxSpeechLevelDbfs);
speech_level_dbfs = rtc::SafeClamp<float>(
speech_level_dbfs, kMinSpeechLevelDbfs, kMaxSpeechLevelDbfs);
int rms_error_db = 0;
if (speech_level_dbfs > target_range_max_dbfs) {
rms_error_db = std::round(target_range_max_dbfs - speech_level_dbfs);
} else if (speech_level_dbfs < target_range_min_dbfs) {
rms_error_db = std::round(target_range_min_dbfs - speech_level_dbfs);
}
return rms_error_db;
}
} // namespace
MonoInputVolumeController::MonoInputVolumeController(
int min_input_volume_after_clipping,
int min_input_volume,
int update_input_volume_wait_frames,
float speech_probability_threshold,
float speech_ratio_threshold)
: min_input_volume_(min_input_volume),
min_input_volume_after_clipping_(min_input_volume_after_clipping),
max_input_volume_(kMaxInputVolume),
update_input_volume_wait_frames_(
std::max(update_input_volume_wait_frames, 1)),
speech_probability_threshold_(speech_probability_threshold),
speech_ratio_threshold_(speech_ratio_threshold) {
RTC_DCHECK_GE(min_input_volume_, 0);
RTC_DCHECK_LE(min_input_volume_, 255);
RTC_DCHECK_GE(min_input_volume_after_clipping_, 0);
RTC_DCHECK_LE(min_input_volume_after_clipping_, 255);
RTC_DCHECK_GE(max_input_volume_, 0);
RTC_DCHECK_LE(max_input_volume_, 255);
RTC_DCHECK_GE(update_input_volume_wait_frames_, 0);
RTC_DCHECK_GE(speech_probability_threshold_, 0.0f);
RTC_DCHECK_LE(speech_probability_threshold_, 1.0f);
RTC_DCHECK_GE(speech_ratio_threshold_, 0.0f);
RTC_DCHECK_LE(speech_ratio_threshold_, 1.0f);
}
MonoInputVolumeController::~MonoInputVolumeController() = default;
void MonoInputVolumeController::Initialize() {
max_input_volume_ = kMaxInputVolume;
capture_output_used_ = true;
check_volume_on_next_process_ = true;
frames_since_update_input_volume_ = 0;
speech_frames_since_update_input_volume_ = 0;
is_first_frame_ = true;
}
// A speeh segment is considered active if at least
// `update_input_volume_wait_frames_` new frames have been processed since the
// previous update and the ratio of non-silence frames (i.e., frames with a
// `speech_probability` higher than `speech_probability_threshold_`) is at least
// `speech_ratio_threshold_`.
void MonoInputVolumeController::Process(std::optional<int> rms_error_db,
float speech_probability) {
if (check_volume_on_next_process_) {
check_volume_on_next_process_ = false;
// We have to wait until the first process call to check the volume,
// because Chromium doesn't guarantee it to be valid any earlier.
CheckVolumeAndReset();
}
// Count frames with a high speech probability as speech.
if (speech_probability >= speech_probability_threshold_) {
++speech_frames_since_update_input_volume_;
}
// Reset the counters and maybe update the input volume.
if (++frames_since_update_input_volume_ >= update_input_volume_wait_frames_) {
const float speech_ratio =
static_cast<float>(speech_frames_since_update_input_volume_) /
static_cast<float>(update_input_volume_wait_frames_);
// Always reset the counters regardless of whether the volume changes or
// not.
frames_since_update_input_volume_ = 0;
speech_frames_since_update_input_volume_ = 0;
// Update the input volume if allowed.
if (!is_first_frame_ && speech_ratio >= speech_ratio_threshold_ &&
rms_error_db.has_value()) {
UpdateInputVolume(*rms_error_db);
}
}
is_first_frame_ = false;
}
void MonoInputVolumeController::HandleClipping(int clipped_level_step) {
RTC_DCHECK_GT(clipped_level_step, 0);
// Always decrease the maximum input volume, even if the current input volume
// is below threshold.
SetMaxLevel(std::max(min_input_volume_after_clipping_,
max_input_volume_ - clipped_level_step));
if (log_to_histograms_) {
RTC_HISTOGRAM_BOOLEAN("WebRTC.Audio.AgcClippingAdjustmentAllowed",
last_recommended_input_volume_ - clipped_level_step >=
min_input_volume_after_clipping_);
}
if (last_recommended_input_volume_ > min_input_volume_after_clipping_) {
// Don't try to adjust the input volume if we're already below the limit. As
// a consequence, if the user has brought the input volume above the limit,
// we will still not react until the postproc updates the input volume.
SetInputVolume(
std::max(min_input_volume_after_clipping_,
last_recommended_input_volume_ - clipped_level_step));
frames_since_update_input_volume_ = 0;
speech_frames_since_update_input_volume_ = 0;
is_first_frame_ = false;
}
}
void MonoInputVolumeController::SetInputVolume(int new_volume) {
int applied_input_volume = recommended_input_volume_;
if (applied_input_volume == 0) {
RTC_DLOG(LS_INFO)
<< "[AGC2] The applied input volume is zero, taking no action.";
return;
}
if (applied_input_volume < 0 || applied_input_volume > kMaxInputVolume) {
RTC_LOG(LS_ERROR) << "[AGC2] Invalid value for the applied input volume: "
<< applied_input_volume;
return;
}
// Detect manual input volume adjustments by checking if the
// `applied_input_volume` is outside of the `[last_recommended_input_volume_ -
// kVolumeQuantizationSlack, last_recommended_input_volume_ +
// kVolumeQuantizationSlack]` range.
if (applied_input_volume >
last_recommended_input_volume_ + kVolumeQuantizationSlack ||
applied_input_volume <
last_recommended_input_volume_ - kVolumeQuantizationSlack) {
RTC_DLOG(LS_INFO)
<< "[AGC2] The input volume was manually adjusted. Updating "
"stored input volume from "
<< last_recommended_input_volume_ << " to " << applied_input_volume;
last_recommended_input_volume_ = applied_input_volume;
// Always allow the user to increase the volume.
if (last_recommended_input_volume_ > max_input_volume_) {
SetMaxLevel(last_recommended_input_volume_);
}
// Take no action in this case, since we can't be sure when the volume
// was manually adjusted.
frames_since_update_input_volume_ = 0;
speech_frames_since_update_input_volume_ = 0;
is_first_frame_ = false;
return;
}
new_volume = std::min(new_volume, max_input_volume_);
if (new_volume == last_recommended_input_volume_) {
return;
}
recommended_input_volume_ = new_volume;
RTC_DLOG(LS_INFO) << "[AGC2] Applied input volume: " << applied_input_volume
<< " | last recommended input volume: "
<< last_recommended_input_volume_
<< " | newly recommended input volume: " << new_volume;
last_recommended_input_volume_ = new_volume;
}
void MonoInputVolumeController::SetMaxLevel(int input_volume) {
RTC_DCHECK_GE(input_volume, min_input_volume_after_clipping_);
max_input_volume_ = input_volume;
RTC_DLOG(LS_INFO) << "[AGC2] Maximum input volume updated: "
<< max_input_volume_;
}
void MonoInputVolumeController::HandleCaptureOutputUsedChange(
bool capture_output_used) {
if (capture_output_used_ == capture_output_used) {
return;
}
capture_output_used_ = capture_output_used;
if (capture_output_used) {
// When we start using the output, we should reset things to be safe.
check_volume_on_next_process_ = true;
}
}
int MonoInputVolumeController::CheckVolumeAndReset() {
int input_volume = recommended_input_volume_;
// Reasons for taking action at startup:
// 1) A person starting a call is expected to be heard.
// 2) Independent of interpretation of `input_volume` == 0 we should raise it
// so the AGC can do its job properly.
if (input_volume == 0 && !startup_) {
RTC_DLOG(LS_INFO)
<< "[AGC2] The applied input volume is zero, taking no action.";
return 0;
}
if (input_volume < 0 || input_volume > kMaxInputVolume) {
RTC_LOG(LS_ERROR) << "[AGC2] Invalid value for the applied input volume: "
<< input_volume;
return -1;
}
RTC_DLOG(LS_INFO) << "[AGC2] Initial input volume: " << input_volume;
if (input_volume < min_input_volume_) {
input_volume = min_input_volume_;
RTC_DLOG(LS_INFO)
<< "[AGC2] The initial input volume is too low, raising to "
<< input_volume;
recommended_input_volume_ = input_volume;
}
last_recommended_input_volume_ = input_volume;
startup_ = false;
frames_since_update_input_volume_ = 0;
speech_frames_since_update_input_volume_ = 0;
is_first_frame_ = true;
return 0;
}
void MonoInputVolumeController::UpdateInputVolume(int rms_error_db) {
RTC_DLOG(LS_INFO) << "[AGC2] RMS error: " << rms_error_db << " dB";
// Prevent too large microphone input volume changes by clamping the RMS
// error.
rms_error_db =
rtc::SafeClamp(rms_error_db, -KMaxAbsRmsErrorDbfs, KMaxAbsRmsErrorDbfs);
if (rms_error_db == 0) {
return;
}
SetInputVolume(ComputeVolumeUpdate(
rms_error_db, last_recommended_input_volume_, min_input_volume_));
}
InputVolumeController::InputVolumeController(int num_capture_channels,
const Config& config)
: num_capture_channels_(num_capture_channels),
min_input_volume_(config.min_input_volume),
capture_output_used_(true),
clipped_level_step_(config.clipped_level_step),
clipped_ratio_threshold_(config.clipped_ratio_threshold),
clipped_wait_frames_(config.clipped_wait_frames),
clipping_predictor_(CreateClippingPredictor(
num_capture_channels,
CreateClippingPredictorConfig(config.enable_clipping_predictor))),
use_clipping_predictor_step_(
!!clipping_predictor_ &&
CreateClippingPredictorConfig(config.enable_clipping_predictor)
.use_predicted_step),
frames_since_clipped_(config.clipped_wait_frames),
clipping_rate_log_counter_(0),
clipping_rate_log_(0.0f),
target_range_max_dbfs_(config.target_range_max_dbfs),
target_range_min_dbfs_(config.target_range_min_dbfs),
channel_controllers_(num_capture_channels) {
RTC_LOG(LS_INFO)
<< "[AGC2] Input volume controller enabled. Minimum input volume: "
<< min_input_volume_;
for (auto& controller : channel_controllers_) {
controller = std::make_unique<MonoInputVolumeController>(
config.clipped_level_min, min_input_volume_,
config.update_input_volume_wait_frames,
config.speech_probability_threshold, config.speech_ratio_threshold);
}
RTC_DCHECK(!channel_controllers_.empty());
RTC_DCHECK_GT(clipped_level_step_, 0);
RTC_DCHECK_LE(clipped_level_step_, 255);
RTC_DCHECK_GT(clipped_ratio_threshold_, 0.0f);
RTC_DCHECK_LT(clipped_ratio_threshold_, 1.0f);
RTC_DCHECK_GT(clipped_wait_frames_, 0);
channel_controllers_[0]->ActivateLogging();
}
InputVolumeController::~InputVolumeController() {}
void InputVolumeController::Initialize() {
for (auto& controller : channel_controllers_) {
controller->Initialize();
}
capture_output_used_ = true;
AggregateChannelLevels();
clipping_rate_log_ = 0.0f;
clipping_rate_log_counter_ = 0;
applied_input_volume_ = std::nullopt;
}
void InputVolumeController::AnalyzeInputAudio(int applied_input_volume,
const AudioBuffer& audio_buffer) {
RTC_DCHECK_GE(applied_input_volume, 0);
RTC_DCHECK_LE(applied_input_volume, 255);
SetAppliedInputVolume(applied_input_volume);
RTC_DCHECK_EQ(audio_buffer.num_channels(), channel_controllers_.size());
const float* const* audio = audio_buffer.channels_const();
size_t samples_per_channel = audio_buffer.num_frames();
RTC_DCHECK(audio);
AggregateChannelLevels();
if (!capture_output_used_) {
return;
}
if (!!clipping_predictor_) {
AudioFrameView<const float> frame = AudioFrameView<const float>(
audio, num_capture_channels_, static_cast<int>(samples_per_channel));
clipping_predictor_->Analyze(frame);
}
// Check for clipped samples. We do this in the preprocessing phase in order
// to catch clipped echo as well.
//
// If we find a sufficiently clipped frame, drop the current microphone
// input volume and enforce a new maximum input volume, dropped the same
// amount from the current maximum. This harsh treatment is an effort to avoid
// repeated clipped echo events.
float clipped_ratio =
ComputeClippedRatio(audio, num_capture_channels_, samples_per_channel);
clipping_rate_log_ = std::max(clipped_ratio, clipping_rate_log_);
clipping_rate_log_counter_++;
constexpr int kNumFramesIn30Seconds = 3000;
if (clipping_rate_log_counter_ == kNumFramesIn30Seconds) {
LogClippingMetrics(std::round(100.0f * clipping_rate_log_));
clipping_rate_log_ = 0.0f;
clipping_rate_log_counter_ = 0;
}
if (frames_since_clipped_ < clipped_wait_frames_) {
++frames_since_clipped_;
return;
}
const bool clipping_detected = clipped_ratio > clipped_ratio_threshold_;
bool clipping_predicted = false;
int predicted_step = 0;
if (!!clipping_predictor_) {
for (int channel = 0; channel < num_capture_channels_; ++channel) {
const auto step = clipping_predictor_->EstimateClippedLevelStep(
channel, recommended_input_volume_, clipped_level_step_,
channel_controllers_[channel]->min_input_volume_after_clipping(),
kMaxInputVolume);
if (step.has_value()) {
predicted_step = std::max(predicted_step, step.value());
clipping_predicted = true;
}
}
}
if (clipping_detected) {
RTC_DLOG(LS_INFO) << "[AGC2] Clipping detected (ratio: " << clipped_ratio
<< ")";
}
int step = clipped_level_step_;
if (clipping_predicted) {
predicted_step = std::max(predicted_step, clipped_level_step_);
RTC_DLOG(LS_INFO) << "[AGC2] Clipping predicted (volume down step: "
<< predicted_step << ")";
if (use_clipping_predictor_step_) {
step = predicted_step;
}
}
if (clipping_detected ||
(clipping_predicted && use_clipping_predictor_step_)) {
for (auto& state_ch : channel_controllers_) {
state_ch->HandleClipping(step);
}
frames_since_clipped_ = 0;
if (!!clipping_predictor_) {
clipping_predictor_->Reset();
}
}
AggregateChannelLevels();
}
std::optional<int> InputVolumeController::RecommendInputVolume(
float speech_probability,
std::optional<float> speech_level_dbfs) {
// Only process if applied input volume is set.
if (!applied_input_volume_.has_value()) {
RTC_LOG(LS_ERROR) << "[AGC2] Applied input volume not set.";
return std::nullopt;
}
AggregateChannelLevels();
const int volume_after_clipping_handling = recommended_input_volume_;
if (!capture_output_used_) {
return applied_input_volume_;
}
std::optional<int> rms_error_db;
if (speech_level_dbfs.has_value()) {
// Compute the error for all frames (both speech and non-speech frames).
rms_error_db = GetSpeechLevelRmsErrorDb(
*speech_level_dbfs, target_range_min_dbfs_, target_range_max_dbfs_);
}
for (auto& controller : channel_controllers_) {
controller->Process(rms_error_db, speech_probability);
}
AggregateChannelLevels();
if (volume_after_clipping_handling != recommended_input_volume_) {
// The recommended input volume was adjusted in order to match the target
// level.
UpdateHistogramOnRecommendedInputVolumeChangeToMatchTarget(
recommended_input_volume_);
}
applied_input_volume_ = std::nullopt;
return recommended_input_volume();
}
void InputVolumeController::HandleCaptureOutputUsedChange(
bool capture_output_used) {
for (auto& controller : channel_controllers_) {
controller->HandleCaptureOutputUsedChange(capture_output_used);
}
capture_output_used_ = capture_output_used;
}
void InputVolumeController::SetAppliedInputVolume(int input_volume) {
applied_input_volume_ = input_volume;
for (auto& controller : channel_controllers_) {
controller->set_stream_analog_level(input_volume);
}
AggregateChannelLevels();
}
void InputVolumeController::AggregateChannelLevels() {
int new_recommended_input_volume =
channel_controllers_[0]->recommended_analog_level();
channel_controlling_gain_ = 0;
for (size_t ch = 1; ch < channel_controllers_.size(); ++ch) {
int input_volume = channel_controllers_[ch]->recommended_analog_level();
if (input_volume < new_recommended_input_volume) {
new_recommended_input_volume = input_volume;
channel_controlling_gain_ = static_cast<int>(ch);
}
}
// Enforce the minimum input volume when a recommendation is made.
if (applied_input_volume_.has_value() && *applied_input_volume_ > 0) {
new_recommended_input_volume =
std::max(new_recommended_input_volume, min_input_volume_);
}
recommended_input_volume_ = new_recommended_input_volume;
}
} // namespace webrtc
View Git Blame Copy Permalink