AnotherFoxGuy/webrtc-audio-processing
1
0
Fork 0
Code Pull Requests Activity

Bump to WebRTC M131 release

Browse Source 
Ongoing fixes and improvements, transient suppressor is gone. Also,
dropping isac because it doesn't seem to be useful, and is just build
system deadweight now.

Upstream references:

  Version: 131.0.6778.200
  WebRTC: 79aff54b0fa9238ce3518dd9eaf9610cd6f22e82
  Chromium: 2a19506ad24af755f2a215a4c61f775393e0db42
This commit is contained in:
Arun Raghavan
2024-12-24 19:32:07 -05:00
parent 8bdb53d91c
commit b5c48b97f6
263 changed files with 4628 additions and 20416 deletions
Download Patch File Download Diff File Expand all files Collapse all files

133
webrtc/modules/audio_processing/BUILD.gn
View File

@ -21,30 +21,8 @@ config("apm_debug_dump") {
rtc_library("api") {
visibility = [ "*" ]
sources = [
"include/audio_processing.cc",
"include/audio_processing.h",
]
deps = [
":audio_frame_view",
":audio_processing_statistics",
"../../api:array_view",
"../../api:scoped_refptr",
"../../api/audio:aec3_config",
"../../api/audio:audio_frame_api",
"../../api/audio:echo_control",
"../../rtc_base:macromagic",
"../../rtc_base:refcount",
"../../rtc_base:stringutils",
"../../rtc_base/system:arch",
"../../rtc_base/system:file_wrapper",
"../../rtc_base/system:rtc_export",
"agc:gain_control_interface",
]
absl_deps = [
"//third_party/abseil-cpp/absl/strings",
"//third_party/abseil-cpp/absl/types:optional",
]
sources = [ "include/audio_processing.h" ]
deps = [ "../../api/audio:audio_processing" ]
}
rtc_library("audio_frame_proxies") {
@ -54,9 +32,9 @@ rtc_library("audio_frame_proxies") {
"include/audio_frame_proxies.h",
]
deps = [
":api",
":audio_frame_view",
"../../api/audio:audio_frame_api",
"../../api/audio:audio_processing",
]
}
@ -77,8 +55,9 @@ rtc_library("audio_buffer") {
defines = []
deps = [
":api",
"../../api:array_view",
"../../api/audio:audio_frame_api",
"../../api/audio:audio_processing",
"../../common_audio",
"../../common_audio:common_audio_c",
"../../rtc_base:checks",
@ -111,12 +90,9 @@ rtc_source_set("aec_dump_interface") {
]
deps = [
":api",
":audio_frame_view",
]
absl_deps = [
"../../api/audio:audio_processing",
"//third_party/abseil-cpp/absl/base:core_headers",
"//third_party/abseil-cpp/absl/types:optional",
]
}
@ -129,10 +105,11 @@ rtc_library("gain_controller2") {
defines = []
deps = [
":aec_dump_interface",
":api",
":apm_logging",
":audio_buffer",
":audio_frame_view",
"../../api/audio:audio_frame_api",
"../../api/audio:audio_processing",
"../../common_audio",
"../../rtc_base:checks",
"../../rtc_base:logging",
@ -168,29 +145,28 @@ rtc_library("audio_processing") {
defines = []
deps = [
":aec_dump_interface",
":api",
":apm_logging",
":audio_buffer",
":audio_frame_proxies",
":audio_frame_view",
":audio_processing_statistics",
":gain_controller2",
":high_pass_filter",
":optionally_built_submodule_creators",
":rms_level",
"../../api:array_view",
"../../api:function_view",
"../../api:make_ref_counted",
"../../api/audio:aec3_config",
"../../api/audio:audio_frame_api",
"../../api/audio:audio_processing",
"../../api/audio:audio_processing_statistics",
"../../api/audio:echo_control",
"../../api/task_queue",
"../../audio/utility:audio_frame_operations",
"../../common_audio:common_audio_c",
"../../common_audio/third_party/ooura:fft_size_256",
"../../rtc_base:checks",
"../../rtc_base:event_tracer",
"../../rtc_base:gtest_prod",
"../../rtc_base:ignore_wundef",
"../../rtc_base:logging",
"../../rtc_base:macromagic",
"../../rtc_base:safe_minmax",
@ -213,12 +189,10 @@ rtc_library("audio_processing") {
"agc2:input_volume_stats_reporter",
"capture_levels_adjuster",
"ns",
"transient:transient_suppressor_api",
"vad",
]
absl_deps = [
"//third_party/abseil-cpp/absl/base:nullability",
"//third_party/abseil-cpp/absl/strings",
"//third_party/abseil-cpp/absl/types:optional",
"//third_party/abseil-cpp/absl/strings:string_view",
]
deps += [
@ -251,25 +225,13 @@ rtc_library("residual_echo_detector") {
"residual_echo_detector.h",
]
deps = [
":api",
":apm_logging",
"../../api:array_view",
"../../api/audio:audio_processing",
"../../rtc_base:checks",
"../../rtc_base:logging",
"../../system_wrappers:metrics",
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}
rtc_library("optionally_built_submodule_creators") {
sources = [
"optionally_built_submodule_creators.cc",
"optionally_built_submodule_creators.h",
]
deps = [
"transient:transient_suppressor_api",
"transient:transient_suppressor_impl",
]
}
rtc_source_set("rms_level") {
@ -282,22 +244,17 @@ rtc_source_set("rms_level") {
"../../api:array_view",
"../../rtc_base:checks",
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}
rtc_library("audio_processing_statistics") {
visibility = [ "*" ]
sources = [
"include/audio_processing_statistics.cc",
"include/audio_processing_statistics.h",
]
deps = [ "../../rtc_base/system:rtc_export" ]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
sources = [ "include/audio_processing_statistics.h" ]
deps = [ "../../api/audio:audio_processing_statistics" ]
}
rtc_source_set("audio_frame_view") {
sources = [ "include/audio_frame_view.h" ]
deps = [ "../../api:array_view" ]
deps = [ "../../api/audio:audio_frame_api" ]
}
if (rtc_enable_protobuf) {
@ -319,10 +276,7 @@ rtc_library("apm_logging") {
"../../common_audio",
"../../rtc_base:checks",
"../../rtc_base:stringutils",
]
absl_deps = [
"//third_party/abseil-cpp/absl/strings",
"//third_party/abseil-cpp/absl/types:optional",
"//third_party/abseil-cpp/absl/strings:string_view",
]
defines = []
}
@ -333,23 +287,21 @@ if (rtc_include_tests) {
sources = [ "include/mock_audio_processing.h" ]
deps = [
":aec_dump_interface",
":api",
":audio_buffer",
":audio_processing",
":audio_processing_statistics",
"../../api/audio:audio_processing",
"../../api/audio:audio_processing_statistics",
"../../api/task_queue",
"../../test:test_support",
"//third_party/abseil-cpp/absl/base:nullability",
"//third_party/abseil-cpp/absl/strings:string_view",
]
absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
}
if (!build_with_chromium) {
group("audio_processing_tests") {
testonly = true
deps = [
":audioproc_test_utils",
"transient:click_annotate",
"transient:transient_suppression_test",
]
deps = [ ":audioproc_test_utils" ]
if (rtc_enable_protobuf) {
deps += [
@ -378,7 +330,6 @@ if (rtc_include_tests) {
deps = [
":aec3_config_json",
":analog_mic_simulation",
":api",
":apm_logging",
":audio_buffer",
":audio_frame_view",
@ -392,12 +343,13 @@ if (rtc_include_tests) {
"../../api:scoped_refptr",
"../../api/audio:aec3_config",
"../../api/audio:aec3_factory",
"../../api/audio:audio_frame_api",
"../../api/audio:audio_processing",
"../../api/audio:echo_detector_creator",
"../../common_audio",
"../../common_audio:common_audio_c",
"../../rtc_base:checks",
"../../rtc_base:gtest_prod",
"../../rtc_base:ignore_wundef",
"../../rtc_base:macromagic",
"../../rtc_base:platform_thread",
"../../rtc_base:protobuf_utils",
@ -437,15 +389,11 @@ if (rtc_include_tests) {
"capture_levels_adjuster",
"capture_levels_adjuster:capture_levels_adjuster_unittests",
"test/conversational_speech:unittest",
"transient:transient_suppression_unittests",
"utility:legacy_delay_estimator_unittest",
"utility:pffft_wrapper_unittest",
"vad:vad_unittests",
"//testing/gtest",
]
absl_deps = [
"//third_party/abseil-cpp/absl/strings",
"//third_party/abseil-cpp/absl/types:optional",
"//third_party/abseil-cpp/absl/strings:string_view",
]
defines = []
@ -463,18 +411,16 @@ if (rtc_include_tests) {
":audioproc_protobuf_utils",
":audioproc_test_utils",
":audioproc_unittest_proto",
":optionally_built_submodule_creators",
":residual_echo_detector",
":rms_level",
":runtime_settings_protobuf_utils",
"../../api/audio:audio_frame_api",
"../../api/audio:echo_control",
"../../rtc_base:rtc_base_tests_utils",
"../../rtc_base:rtc_task_queue",
"aec_dump",
"aec_dump:aec_dump_unittests",
"//third_party/abseil-cpp/absl/flags:flag",
]
absl_deps += [ "//third_party/abseil-cpp/absl/flags:flag" ]
sources += [
"audio_processing_impl_locking_unittest.cc",
"audio_processing_impl_unittest.cc",
@ -520,8 +466,8 @@ if (rtc_include_tests) {
"../../rtc_base:safe_conversions",
"../../system_wrappers",
"../../test:test_support",
"//third_party/abseil-cpp/absl/strings:string_view",
]
absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
}
rtc_library("analog_mic_simulation") {
@ -539,7 +485,6 @@ if (rtc_include_tests) {
"../../rtc_base:safe_minmax",
"agc2:gain_map",
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}
if (rtc_enable_protobuf && !build_with_chromium) {
@ -562,7 +507,6 @@ if (rtc_include_tests) {
deps = [
":aec3_config_json",
":analog_mic_simulation",
":api",
":apm_logging",
":audio_processing",
":audioproc_debug_proto",
@ -570,10 +514,10 @@ if (rtc_include_tests) {
":audioproc_test_utils",
":runtime_settings_protobuf_utils",
"../../api/audio:aec3_factory",
"../../api/audio:audio_processing",
"../../api/audio:echo_detector_creator",
"../../common_audio",
"../../rtc_base:checks",
"../../rtc_base:ignore_wundef",
"../../rtc_base:logging",
"../../rtc_base:protobuf_utils",
"../../rtc_base:rtc_json",
@ -588,12 +532,9 @@ if (rtc_include_tests) {
"aec_dump",
"aec_dump:aec_dump_impl",
"//testing/gtest",
]
absl_deps = [
"//third_party/abseil-cpp/absl/flags:flag",
"//third_party/abseil-cpp/absl/flags:parse",
"//third_party/abseil-cpp/absl/strings",
"//third_party/abseil-cpp/absl/types:optional",
"//third_party/abseil-cpp/absl/strings:string_view",
]
} # audioproc_f_impl
}
@ -613,7 +554,6 @@ if (rtc_include_tests) {
deps = [
":audioproc_debug_proto",
"../../rtc_base:checks",
"../../rtc_base:ignore_wundef",
"../../rtc_base:protobuf_utils",
"../../rtc_base/system:arch",
]
@ -627,9 +567,9 @@ if (rtc_include_tests) {
]
deps = [
":api",
":audioproc_debug_proto",
":audioproc_protobuf_utils",
"../../api/audio:audio_processing",
"../../rtc_base:checks",
]
}
@ -657,11 +597,11 @@ rtc_library("audioproc_test_utils") {
configs += [ ":apm_debug_dump" ]
deps = [
":api",
":audio_buffer",
":audio_processing",
"../../api:array_view",
"../../api/audio:audio_frame_api",
"../../api/audio:audio_processing",
"../../common_audio",
"../../rtc_base:checks",
"../../rtc_base:random",
@ -671,10 +611,7 @@ rtc_library("audioproc_test_utils") {
"../../test:test_support",
"../audio_coding:neteq_input_audio_tools",
"//testing/gtest",
]
absl_deps = [
"//third_party/abseil-cpp/absl/strings",
"//third_party/abseil-cpp/absl/types:optional",
"//third_party/abseil-cpp/absl/strings:string_view",
]
}
@ -692,6 +629,6 @@ rtc_library("aec3_config_json") {
"../../rtc_base:rtc_json",
"../../rtc_base:stringutils",
"../../rtc_base/system:rtc_export",
"//third_party/abseil-cpp/absl/strings:string_view",
]
absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
}

9
webrtc/modules/audio_processing/aec3/BUILD.gn
View File

@ -156,10 +156,7 @@ rtc_library("aec3") {
"../../../system_wrappers:field_trial",
"../../../system_wrappers:metrics",
"../utility:cascaded_biquad_filter",
]
absl_deps = [
"//third_party/abseil-cpp/absl/strings",
"//third_party/abseil-cpp/absl/types:optional",
"//third_party/abseil-cpp/absl/strings:string_view",
]
if (current_cpu == "x86" || current_cpu == "x64") {
@ -210,8 +207,8 @@ rtc_source_set("adaptive_fir_filter") {
"..:apm_logging",
"../../../api:array_view",
"../../../rtc_base/system:arch",
"//third_party/abseil-cpp/absl/strings:string_view",
]
absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
}
rtc_source_set("adaptive_fir_filter_erl") {
@ -231,7 +228,6 @@ rtc_source_set("matched_filter") {
"../../../rtc_base:gtest_prod",
"../../../rtc_base/system:arch",
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}
rtc_source_set("vector_math") {
@ -329,7 +325,6 @@ if (rtc_include_tests) {
"../../../test:test_support",
"../utility:cascaded_biquad_filter",
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
defines = []

8
webrtc/modules/audio_processing/aec3/aec_state.cc
View File

@ -14,9 +14,9 @@
#include <algorithm>
#include <numeric>
#include <optional>
#include <vector>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/logging/apm_data_dumper.h"
@ -170,7 +170,7 @@ void AecState::HandleEchoPathChange(
}
void AecState::Update(
const absl::optional<DelayEstimate>& external_delay,
const std::optional<DelayEstimate>& external_delay,
rtc::ArrayView<const std::vector<std::array<float, kFftLengthBy2Plus1>>>
adaptive_filter_frequency_responses,
rtc::ArrayView<const std::vector<float>> adaptive_filter_impulse_responses,
@ -359,7 +359,7 @@ AecState::FilterDelay::FilterDelay(const EchoCanceller3Config& config,
void AecState::FilterDelay::Update(
rtc::ArrayView<const int> analyzer_filter_delay_estimates_blocks,
const absl::optional<DelayEstimate>& external_delay,
const std::optional<DelayEstimate>& external_delay,
size_t blocks_with_proper_filter_adaptation) {
// Update the delay based on the external delay.
if (external_delay &&
@ -405,7 +405,7 @@ void AecState::FilteringQualityAnalyzer::Update(
bool active_render,
bool transparent_mode,
bool saturated_capture,
const absl::optional<DelayEstimate>& external_delay,
const std::optional<DelayEstimate>& external_delay,
bool any_filter_converged) {
// Update blocks counter.
const bool filter_update = active_render && !saturated_capture;

10
webrtc/modules/audio_processing/aec3/aec_state.h
View File

@ -16,9 +16,9 @@
#include <array>
#include <atomic>
#include <memory>
#include <optional>
#include <vector>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/audio/echo_canceller3_config.h"
#include "modules/audio_processing/aec3/aec3_common.h"
@ -138,7 +138,7 @@ class AecState {
// Updates the aec state.
// TODO(bugs.webrtc.org/10913): Compute multi-channel ERL.
void Update(
const absl::optional<DelayEstimate>& external_delay,
const std::optional<DelayEstimate>& external_delay,
rtc::ArrayView<const std::vector<std::array<float, kFftLengthBy2Plus1>>>
adaptive_filter_frequency_responses,
rtc::ArrayView<const std::vector<float>>
@ -213,7 +213,7 @@ class AecState {
// Updates the delay estimates based on new data.
void Update(
rtc::ArrayView<const int> analyzer_filter_delay_estimates_blocks,
const absl::optional<DelayEstimate>& external_delay,
const std::optional<DelayEstimate>& external_delay,
size_t blocks_with_proper_filter_adaptation);
private:
@ -221,7 +221,7 @@ class AecState {
bool external_delay_reported_ = false;
std::vector<int> filter_delays_blocks_;
int min_filter_delay_;
absl::optional<DelayEstimate> external_delay_;
std::optional<DelayEstimate> external_delay_;
} delay_state_;
// Classifier for toggling transparent mode when there is no echo.
@ -253,7 +253,7 @@ class AecState {
void Update(bool active_render,
bool transparent_mode,
bool saturated_capture,
const absl::optional<DelayEstimate>& external_delay,
const std::optional<DelayEstimate>& external_delay,
bool any_filter_converged);
private:

6
webrtc/modules/audio_processing/aec3/block_processor.cc
View File

@ -13,10 +13,10 @@
#include <atomic>
#include <memory>
#include <optional>
#include <utility>
#include <vector>
#include "absl/types/optional.h"
#include "api/audio/echo_canceller3_config.h"
#include "api/audio/echo_control.h"
#include "modules/audio_processing/aec3/aec3_common.h"
@ -76,7 +76,7 @@ class BlockProcessorImpl final : public BlockProcessor {
BlockProcessorMetrics metrics_;
RenderDelayBuffer::BufferingEvent render_event_;
size_t capture_call_counter_ = 0;
absl::optional<DelayEstimate> estimated_delay_;
std::optional<DelayEstimate> estimated_delay_;
};
std::atomic<int> BlockProcessorImpl::instance_count_(0);
@ -223,7 +223,7 @@ void BlockProcessorImpl::UpdateEchoLeakageStatus(bool leakage_detected) {
void BlockProcessorImpl::GetMetrics(EchoControl::Metrics* metrics) const {
echo_remover_->GetMetrics(metrics);
constexpr int block_size_ms = 4;
absl::optional<size_t> delay = render_buffer_->Delay();
std::optional<size_t> delay = render_buffer_->Delay();
metrics->delay_ms = delay ? static_cast<int>(*delay) * block_size_ms : 0;
}

2
webrtc/modules/audio_processing/aec3/config_selector.cc
View File

@ -47,7 +47,7 @@ bool CompatibleConfigs(const EchoCanceller3Config& mono_config,
ConfigSelector::ConfigSelector(
const EchoCanceller3Config& config,
const absl::optional<EchoCanceller3Config>& multichannel_config,
const std::optional<EchoCanceller3Config>& multichannel_config,
int num_render_input_channels)
: config_(config), multichannel_config_(multichannel_config) {
if (multichannel_config_.has_value()) {

12
webrtc/modules/audio_processing/aec3/config_selector.h
View File

@ -11,7 +11,8 @@
#ifndef MODULES_AUDIO_PROCESSING_AEC3_CONFIG_SELECTOR_H_
#define MODULES_AUDIO_PROCESSING_AEC3_CONFIG_SELECTOR_H_
#include "absl/types/optional.h"
#include <optional>
#include "api/audio/echo_canceller3_config.h"
namespace webrtc {
@ -19,10 +20,9 @@ namespace webrtc {
// Selects the config to use.
class ConfigSelector {
public:
ConfigSelector(
const EchoCanceller3Config& config,
const absl::optional<EchoCanceller3Config>& multichannel_config,
int num_render_input_channels);
ConfigSelector(const EchoCanceller3Config& config,
const std::optional<EchoCanceller3Config>& multichannel_config,
int num_render_input_channels);
// Updates the config selection based on the detection of multichannel
// content.
@ -32,7 +32,7 @@ class ConfigSelector {
private:
const EchoCanceller3Config config_;
const absl::optional<EchoCanceller3Config> multichannel_config_;
const std::optional<EchoCanceller3Config> multichannel_config_;
const EchoCanceller3Config* active_config_ = nullptr;
};

2
webrtc/modules/audio_processing/aec3/echo_audibility.cc
View File

@ -45,7 +45,7 @@ void EchoAudibility::Update(const RenderBuffer& render_buffer,
void EchoAudibility::Reset() {
render_stationarity_.Reset();
non_zero_render_seen_ = false;
render_spectrum_write_prev_ = absl::nullopt;
render_spectrum_write_prev_ = std::nullopt;
}
void EchoAudibility::UpdateRenderStationarityFlags(

5
webrtc/modules/audio_processing/aec3/echo_audibility.h
View File

@ -13,7 +13,8 @@
#include <stddef.h>
#include "absl/types/optional.h"
#include <optional>
#include "api/array_view.h"
#include "modules/audio_processing/aec3/block_buffer.h"
#include "modules/audio_processing/aec3/render_buffer.h"
@ -73,7 +74,7 @@ class EchoAudibility {
// values.
bool IsRenderTooLow(const BlockBuffer& block_buffer);
absl::optional<int> render_spectrum_write_prev_;
std::optional<int> render_spectrum_write_prev_;
int render_block_write_prev_;
bool non_zero_render_seen_;
const bool use_render_stationarity_at_init_;

17
webrtc/modules/audio_processing/aec3/echo_canceller3.cc
View File

@ -378,14 +378,6 @@ EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) {
false;
}
if (field_trial::IsEnabled("WebRTC-Aec3DelayEstimatorDetectPreEcho")) {
adjusted_cfg.delay.detect_pre_echo = true;
}
if (field_trial::IsDisabled("WebRTC-Aec3DelayEstimatorDetectPreEcho")) {
adjusted_cfg.delay.detect_pre_echo = false;
}
if (field_trial::IsEnabled("WebRTC-Aec3SensitiveDominantNearendActivation")) {
adjusted_cfg.suppressor.dominant_nearend_detection.enr_threshold = 0.5f;
} else if (field_trial::IsEnabled(
@ -641,6 +633,13 @@ EchoCanceller3Config AdjustConfig(const EchoCanceller3Config& config) {
"WebRTC-Aec3DelayEstimateSmoothingDelayFoundOverride", 0.f, 1.f,
&adjusted_cfg.delay.delay_estimate_smoothing_delay_found);
int max_allowed_excess_render_blocks_override =
adjusted_cfg.buffering.max_allowed_excess_render_blocks;
RetrieveFieldTrialValue(
"WebRTC-Aec3BufferingMaxAllowedExcessRenderBlocksOverride", 0, 20,
&max_allowed_excess_render_blocks_override);
adjusted_cfg.buffering.max_allowed_excess_render_blocks =
max_allowed_excess_render_blocks_override;
return adjusted_cfg;
}
@ -719,7 +718,7 @@ std::atomic<int> EchoCanceller3::instance_count_(0);
EchoCanceller3::EchoCanceller3(
const EchoCanceller3Config& config,
const absl::optional<EchoCanceller3Config>& multichannel_config,
const std::optional<EchoCanceller3Config>& multichannel_config,
int sample_rate_hz,
size_t num_render_channels,
size_t num_capture_channels)

13
webrtc/modules/audio_processing/aec3/echo_canceller3.h
View File

@ -15,9 +15,9 @@
#include <atomic>
#include <memory>
#include <optional>
#include <vector>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/audio/echo_canceller3_config.h"
#include "api/audio/echo_control.h"
@ -88,12 +88,11 @@ class Aec3RenderQueueItemVerifier {
// AnalyzeRender call which can be called concurrently with the other methods.
class EchoCanceller3 : public EchoControl {
public:
EchoCanceller3(
const EchoCanceller3Config& config,
const absl::optional<EchoCanceller3Config>& multichannel_config,
int sample_rate_hz,
size_t num_render_channels,
size_t num_capture_channels);
EchoCanceller3(const EchoCanceller3Config& config,
const std::optional<EchoCanceller3Config>& multichannel_config,
int sample_rate_hz,
size_t num_render_channels,
size_t num_capture_channels);
~EchoCanceller3() override;

6
webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.cc
View File

@ -58,7 +58,7 @@ void EchoPathDelayEstimator::Reset(bool reset_delay_confidence) {
Reset(true, reset_delay_confidence);
}
absl::optional<DelayEstimate> EchoPathDelayEstimator::EstimateDelay(
std::optional<DelayEstimate> EchoPathDelayEstimator::EstimateDelay(
const DownsampledRenderBuffer& render_buffer,
const Block& capture) {
std::array<float, kBlockSize> downsampled_capture_data;
@ -74,7 +74,7 @@ absl::optional<DelayEstimate> EchoPathDelayEstimator::EstimateDelay(
matched_filter_.Update(render_buffer, downsampled_capture,
matched_filter_lag_aggregator_.ReliableDelayFound());
absl::optional<DelayEstimate> aggregated_matched_filter_lag =
std::optional<DelayEstimate> aggregated_matched_filter_lag =
matched_filter_lag_aggregator_.Aggregate(
matched_filter_.GetBestLagEstimate());
@ -121,7 +121,7 @@ void EchoPathDelayEstimator::Reset(bool reset_lag_aggregator,
matched_filter_lag_aggregator_.Reset(reset_delay_confidence);
}
matched_filter_.Reset(/*full_reset=*/reset_lag_aggregator);
old_aggregated_lag_ = absl::nullopt;
old_aggregated_lag_ = std::nullopt;
consistent_estimate_counter_ = 0;
}
} // namespace webrtc

7
webrtc/modules/audio_processing/aec3/echo_path_delay_estimator.h
View File

@ -13,7 +13,8 @@
#include <stddef.h>
#include "absl/types/optional.h"
#include <optional>
#include "api/array_view.h"
#include "modules/audio_processing/aec3/alignment_mixer.h"
#include "modules/audio_processing/aec3/block.h"
@ -45,7 +46,7 @@ class EchoPathDelayEstimator {
void Reset(bool reset_delay_confidence);
// Produce a delay estimate if such is avaliable.
absl::optional<DelayEstimate> EstimateDelay(
std::optional<DelayEstimate> EstimateDelay(
const DownsampledRenderBuffer& render_buffer,
const Block& capture);
@ -68,7 +69,7 @@ class EchoPathDelayEstimator {
Decimator capture_decimator_;
MatchedFilter matched_filter_;
MatchedFilterLagAggregator matched_filter_lag_aggregator_;
absl::optional<DelayEstimate> old_aggregated_lag_;
std::optional<DelayEstimate> old_aggregated_lag_;
size_t consistent_estimate_counter_ = 0;
ClockdriftDetector clockdrift_detector_;

4
webrtc/modules/audio_processing/aec3/echo_remover.cc
View File

@ -120,7 +120,7 @@ class EchoRemoverImpl final : public EchoRemover {
// signal.
void ProcessCapture(EchoPathVariability echo_path_variability,
bool capture_signal_saturation,
const absl::optional<DelayEstimate>& external_delay,
const std::optional<DelayEstimate>& external_delay,
RenderBuffer* render_buffer,
Block* linear_output,
Block* capture) override;
@ -239,7 +239,7 @@ void EchoRemoverImpl::GetMetrics(EchoControl::Metrics* metrics) const {
void EchoRemoverImpl::ProcessCapture(
EchoPathVariability echo_path_variability,
bool capture_signal_saturation,
const absl::optional<DelayEstimate>& external_delay,
const std::optional<DelayEstimate>& external_delay,
RenderBuffer* render_buffer,
Block* linear_output,
Block* capture) {

4
webrtc/modules/audio_processing/aec3/echo_remover.h
View File

@ -11,9 +11,9 @@
#ifndef MODULES_AUDIO_PROCESSING_AEC3_ECHO_REMOVER_H_
#define MODULES_AUDIO_PROCESSING_AEC3_ECHO_REMOVER_H_
#include <optional>
#include <vector>
#include "absl/types/optional.h"
#include "api/audio/echo_canceller3_config.h"
#include "api/audio/echo_control.h"
#include "modules/audio_processing/aec3/block.h"
@ -41,7 +41,7 @@ class EchoRemover {
virtual void ProcessCapture(
EchoPathVariability echo_path_variability,
bool capture_signal_saturation,
const absl::optional<DelayEstimate>& external_delay,
const std::optional<DelayEstimate>& external_delay,
RenderBuffer* render_buffer,
Block* linear_output,
Block* capture) = 0;

4
webrtc/modules/audio_processing/aec3/erle_estimator.h
View File

@ -15,9 +15,9 @@
#include <array>
#include <memory>
#include <optional>
#include <vector>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/audio/echo_canceller3_config.h"
#include "modules/audio_processing/aec3/aec3_common.h"
@ -91,7 +91,7 @@ class ErleEstimator {
// vector with content between 0 and 1 where 1 indicates that, at this current
// time instant, the linear filter is reaching its maximum subtraction
// performance.
rtc::ArrayView<const absl::optional<float>> GetInstLinearQualityEstimates()
rtc::ArrayView<const std::optional<float>> GetInstLinearQualityEstimates()
const {
return fullband_erle_estimator_.GetInstLinearQualityEstimates();
}

4
webrtc/modules/audio_processing/aec3/fullband_erle_estimator.cc
View File

@ -13,8 +13,8 @@
#include <algorithm>
#include <memory>
#include <numeric>
#include <optional>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/logging/apm_data_dumper.h"
@ -142,7 +142,7 @@ void FullBandErleEstimator::ErleInstantaneous::Reset() {
}
void FullBandErleEstimator::ErleInstantaneous::ResetAccumulators() {
erle_log2_ = absl::nullopt;
erle_log2_ = std::nullopt;
inst_quality_estimate_ = 0.f;
num_points_ = 0;
E2_acum_ = 0.f;

16
webrtc/modules/audio_processing/aec3/fullband_erle_estimator.h
View File

@ -12,9 +12,9 @@
#define MODULES_AUDIO_PROCESSING_AEC3_FULLBAND_ERLE_ESTIMATOR_H_
#include <memory>
#include <optional>
#include <vector>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/audio/echo_canceller3_config.h"
#include "modules/audio_processing/aec3/aec3_common.h"
@ -49,7 +49,7 @@ class FullBandErleEstimator {
// Returns an estimation of the current linear filter quality. It returns a
// float number between 0 and 1 mapping 1 to the highest possible quality.
rtc::ArrayView<const absl::optional<float>> GetInstLinearQualityEstimates()
rtc::ArrayView<const std::optional<float>> GetInstLinearQualityEstimates()
const {
return linear_filters_qualities_;
}
@ -73,10 +73,10 @@ class FullBandErleEstimator {
// Resets the members related with an instantaneous estimate.
void ResetAccumulators();
// Returns the instantaneous ERLE in log2 units.
absl::optional<float> GetInstErleLog2() const { return erle_log2_; }
std::optional<float> GetInstErleLog2() const { return erle_log2_; }
// Gets an indication between 0 and 1 of the performance of the linear
// filter for the current time instant.
absl::optional<float> GetQualityEstimate() const {
std::optional<float> GetQualityEstimate() const {
if (erle_log2_) {
float value = inst_quality_estimate_;
if (clamp_inst_quality_to_zero_) {
@ -85,9 +85,9 @@ class FullBandErleEstimator {
if (clamp_inst_quality_to_one_) {
value = std::min(1.f, value);
}
return absl::optional<float>(value);
return std::optional<float>(value);
}
return absl::nullopt;
return std::nullopt;
}
void Dump(const std::unique_ptr<ApmDataDumper>& data_dumper) const;
@ -96,7 +96,7 @@ class FullBandErleEstimator {
void UpdateQualityEstimate();
const bool clamp_inst_quality_to_zero_;
const bool clamp_inst_quality_to_one_;
absl::optional<float> erle_log2_;
std::optional<float> erle_log2_;
float inst_quality_estimate_;
float max_erle_log2_;
float min_erle_log2_;
@ -110,7 +110,7 @@ class FullBandErleEstimator {
std::vector<int> hold_counters_instantaneous_erle_;
std::vector<float> erle_time_domain_log2_;
std::vector<ErleInstantaneous> instantaneous_erle_;
std::vector<absl::optional<float>> linear_filters_qualities_;
std::vector<std::optional<float>> linear_filters_qualities_;
};
} // namespace webrtc

127
webrtc/modules/audio_processing/aec3/matched_filter.cc
View File

@ -23,8 +23,8 @@
#include <initializer_list>
#include <iterator>
#include <numeric>
#include <optional>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "modules/audio_processing/aec3/downsampled_render_buffer.h"
#include "modules/audio_processing/logging/apm_data_dumper.h"
@ -43,8 +43,8 @@ constexpr int kAccumulatedErrorSubSampleRate = 4;
void UpdateAccumulatedError(
const rtc::ArrayView<const float> instantaneous_accumulated_error,
const rtc::ArrayView<float> accumulated_error,
float one_over_error_sum_anchor,
float smooth_constant_increases) {
float one_over_error_sum_anchor) {
static constexpr float kSmoothConstantIncreases = 0.015f;
for (size_t k = 0; k < instantaneous_accumulated_error.size(); ++k) {
float error_norm =
instantaneous_accumulated_error[k] * one_over_error_sum_anchor;
@ -52,97 +52,30 @@ void UpdateAccumulatedError(
accumulated_error[k] = error_norm;
} else {
accumulated_error[k] +=
smooth_constant_increases * (error_norm - accumulated_error[k]);
kSmoothConstantIncreases * (error_norm - accumulated_error[k]);
}
}
}
size_t ComputePreEchoLag(
const webrtc::MatchedFilter::PreEchoConfiguration& pre_echo_configuration,
const rtc::ArrayView<const float> accumulated_error,
size_t lag,
size_t alignment_shift_winner) {
static constexpr float kPreEchoThreshold = 0.5f;
RTC_DCHECK_GE(lag, alignment_shift_winner);
size_t pre_echo_lag_estimate = lag - alignment_shift_winner;
size_t maximum_pre_echo_lag =
std::min(pre_echo_lag_estimate / kAccumulatedErrorSubSampleRate,
accumulated_error.size());
switch (pre_echo_configuration.mode) {
case 0:
// Mode 0: Pre echo lag is defined as the first coefficient with an error
// lower than a threshold with a certain decrease slope.
for (size_t k = 1; k < maximum_pre_echo_lag; ++k) {
if (accumulated_error[k] <
pre_echo_configuration.threshold * accumulated_error[k - 1] &&
accumulated_error[k] < pre_echo_configuration.threshold) {
pre_echo_lag_estimate = (k + 1) * kAccumulatedErrorSubSampleRate - 1;
break;
}
}
break;
case 1:
// Mode 1: Pre echo lag is defined as the first coefficient with an error
// lower than a certain threshold.
for (size_t k = 0; k < maximum_pre_echo_lag; ++k) {
if (accumulated_error[k] < pre_echo_configuration.threshold) {
pre_echo_lag_estimate = (k + 1) * kAccumulatedErrorSubSampleRate - 1;
break;
}
}
break;
case 2:
case 3:
// Mode 2,3: Pre echo lag is defined as the closest coefficient to the lag
// with an error lower than a certain threshold.
for (int k = static_cast<int>(maximum_pre_echo_lag) - 1; k >= 0; --k) {
if (accumulated_error[k] > pre_echo_configuration.threshold) {
break;
}
pre_echo_lag_estimate = (k + 1) * kAccumulatedErrorSubSampleRate - 1;
}
break;
default:
RTC_DCHECK_NOTREACHED();
for (int k = static_cast<int>(maximum_pre_echo_lag) - 1; k >= 0; --k) {
if (accumulated_error[k] > kPreEchoThreshold) {
break;
}
pre_echo_lag_estimate = (k + 1) * kAccumulatedErrorSubSampleRate - 1;
}
return pre_echo_lag_estimate + alignment_shift_winner;
}
webrtc::MatchedFilter::PreEchoConfiguration FetchPreEchoConfiguration() {
constexpr float kDefaultThreshold = 0.5f;
constexpr int kDefaultMode = 3;
float threshold = kDefaultThreshold;
int mode = kDefaultMode;
const std::string pre_echo_configuration_field_trial =
webrtc::field_trial::FindFullName("WebRTC-Aec3PreEchoConfiguration");
webrtc::FieldTrialParameter<double> threshold_field_trial_parameter(
/*key=*/"threshold", /*default_value=*/kDefaultThreshold);
webrtc::FieldTrialParameter<int> mode_field_trial_parameter(
/*key=*/"mode", /*default_value=*/kDefaultMode);
webrtc::ParseFieldTrial(
{&threshold_field_trial_parameter, &mode_field_trial_parameter},
pre_echo_configuration_field_trial);
float threshold_read =
static_cast<float>(threshold_field_trial_parameter.Get());
int mode_read = mode_field_trial_parameter.Get();
if (threshold_read < 1.0f && threshold_read > 0.0f) {
threshold = threshold_read;
} else {
RTC_LOG(LS_ERROR)
<< "AEC3: Pre echo configuration: wrong input, threshold = "
<< threshold_read << ".";
}
if (mode_read >= 0 && mode_read <= 3) {
mode = mode_read;
} else {
RTC_LOG(LS_ERROR) << "AEC3: Pre echo configuration: wrong input, mode = "
<< mode_read << ".";
}
RTC_LOG(LS_INFO) << "AEC3: Pre echo configuration: threshold = " << threshold
<< ", mode = " << mode << ".";
return {.threshold = threshold, .mode = mode};
}
} // namespace
namespace webrtc {
@ -685,8 +618,7 @@ MatchedFilter::MatchedFilter(ApmDataDumper* data_dumper,
smoothing_fast_(smoothing_fast),
smoothing_slow_(smoothing_slow),
matching_filter_threshold_(matching_filter_threshold),
detect_pre_echo_(detect_pre_echo),
pre_echo_config_(FetchPreEchoConfiguration()) {
detect_pre_echo_(detect_pre_echo) {
RTC_DCHECK(data_dumper);
RTC_DCHECK_LT(0, window_size_sub_blocks);
RTC_DCHECK((kBlockSize % sub_block_size) == 0);
@ -715,9 +647,9 @@ void MatchedFilter::Reset(bool full_reset) {
std::fill(f.begin(), f.end(), 0.f);
}
winner_lag_ = absl::nullopt;
reported_lag_estimate_ = absl::nullopt;
if (pre_echo_config_.mode != 3 || full_reset) {
winner_lag_ = std::nullopt;
reported_lag_estimate_ = std::nullopt;
if (full_reset) {
for (auto& e : accumulated_error_) {
std::fill(e.begin(), e.end(), 1.0f);
}
@ -745,10 +677,10 @@ void MatchedFilter::Update(const DownsampledRenderBuffer& render_buffer,
// Apply all matched filters.
float winner_error_sum = error_sum_anchor;
winner_lag_ = absl::nullopt;
reported_lag_estimate_ = absl::nullopt;
winner_lag_ = std::nullopt;
reported_lag_estimate_ = std::nullopt;
size_t alignment_shift = 0;
absl::optional<size_t> previous_lag_estimate;
std::optional<size_t> previous_lag_estimate;
const int num_filters = static_cast<int>(filters_.size());
int winner_index = -1;
for (int n = 0; n < num_filters; ++n) {
@ -823,22 +755,16 @@ void MatchedFilter::Update(const DownsampledRenderBuffer& render_buffer,
reported_lag_estimate_ =
LagEstimate(winner_lag_.value(), /*pre_echo_lag=*/winner_lag_.value());
if (detect_pre_echo_ && last_detected_best_lag_filter_ == winner_index) {
const float energy_threshold =
pre_echo_config_.mode == 3 ? 1.0f : 30.0f * 30.0f * y.size();
if (error_sum_anchor > energy_threshold) {
const float smooth_constant_increases =
pre_echo_config_.mode != 3 ? 0.01f : 0.015f;
UpdateAccumulatedError(
instantaneous_accumulated_error_, accumulated_error_[winner_index],
1.0f / error_sum_anchor, smooth_constant_increases);
static constexpr float kEnergyThreshold = 1.0f;
if (error_sum_anchor > kEnergyThreshold) {
UpdateAccumulatedError(instantaneous_accumulated_error_,
accumulated_error_[winner_index],
1.0f / error_sum_anchor);
number_pre_echo_updates_++;
}
if (pre_echo_config_.mode != 3 || number_pre_echo_updates_ >= 50) {
if (number_pre_echo_updates_ >= 50) {
reported_lag_estimate_->pre_echo_lag = ComputePreEchoLag(
pre_echo_config_, accumulated_error_[winner_index],
winner_lag_.value(),
accumulated_error_[winner_index], winner_lag_.value(),
winner_index * filter_intra_lag_shift_ /*alignment_shift_winner*/);
} else {
reported_lag_estimate_->pre_echo_lag = winner_lag_.value();
@ -885,10 +811,9 @@ void MatchedFilter::Dump() {
"aec3_correlator_error_" + std::to_string(n) + "_h";
data_dumper_->DumpRaw(dumper_error.c_str(), accumulated_error_[n]);
size_t pre_echo_lag =
ComputePreEchoLag(pre_echo_config_, accumulated_error_[n],
lag_estimate + n * filter_intra_lag_shift_,
n * filter_intra_lag_shift_);
size_t pre_echo_lag = ComputePreEchoLag(
accumulated_error_[n], lag_estimate + n * filter_intra_lag_shift_,
n * filter_intra_lag_shift_);
std::string dumper_pre_lag =
"aec3_correlator_pre_echo_lag_" + std::to_string(n);
data_dumper_->DumpRaw(dumper_pre_lag.c_str(), pre_echo_lag);

23
webrtc/modules/audio_processing/aec3/matched_filter.h
View File

@ -13,9 +13,9 @@
#include <stddef.h>
#include <optional>
#include <vector>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "rtc_base/gtest_prod_util.h"
@ -106,11 +106,6 @@ class MatchedFilter {
size_t pre_echo_lag = 0;
};
struct PreEchoConfiguration {
const float threshold;
const int mode;
};
MatchedFilter(ApmDataDumper* data_dumper,
Aec3Optimization optimization,
size_t sub_block_size,
@ -138,7 +133,7 @@ class MatchedFilter {
void Reset(bool full_reset);
// Returns the current lag estimates.
absl::optional<const MatchedFilter::LagEstimate> GetBestLagEstimate() const {
std::optional<const MatchedFilter::LagEstimate> GetBestLagEstimate() const {
return reported_lag_estimate_;
}
@ -153,15 +148,6 @@ class MatchedFilter {
size_t downsampling_factor) const;
private:
FRIEND_TEST_ALL_PREFIXES(MatchedFilterFieldTrialTest,
PreEchoConfigurationTest);
FRIEND_TEST_ALL_PREFIXES(MatchedFilterFieldTrialTest,
WrongPreEchoConfigurationTest);
// Only for testing. Gets the pre echo detection configuration.
const PreEchoConfiguration& GetPreEchoConfiguration() const {
return pre_echo_config_;
}
void Dump();
ApmDataDumper* const data_dumper_;
@ -172,8 +158,8 @@ class MatchedFilter {
std::vector<std::vector<float>> accumulated_error_;
std::vector<float> instantaneous_accumulated_error_;
std::vector<float> scratch_memory_;
absl::optional<MatchedFilter::LagEstimate> reported_lag_estimate_;
absl::optional<size_t> winner_lag_;
std::optional<MatchedFilter::LagEstimate> reported_lag_estimate_;
std::optional<size_t> winner_lag_;
int last_detected_best_lag_filter_ = -1;
std::vector<size_t> filters_offsets_;
int number_pre_echo_updates_ = 0;
@ -182,7 +168,6 @@ class MatchedFilter {
const float smoothing_slow_;
const float matching_filter_threshold_;
const bool detect_pre_echo_;
const PreEchoConfiguration pre_echo_config_;
};
} // namespace webrtc

30
webrtc/modules/audio_processing/aec3/matched_filter_avx2.cc
View File

@ -13,16 +13,6 @@
#include "modules/audio_processing/aec3/matched_filter.h"
#include "rtc_base/checks.h"
#ifdef _MSC_VER
// Visual Studio
#define LOOKUP_M128(v, i) v.m128_f32[i]
#define LOOKUP_M256(v, i) v.m256_f32[i]
#else
// GCC/Clang
#define LOOKUP_M128(v, i) v[i]
#define LOOKUP_M256(v, i) v[i]
#endif
namespace webrtc {
namespace aec3 {
@ -91,14 +81,14 @@ void MatchedFilterCore_AccumulatedError_AVX2(
s_inst_256_8 = _mm256_mul_ps(h_k_8, x_k_8);
s_inst_hadd_256 = _mm256_hadd_ps(s_inst_256, s_inst_256_8);
s_inst_hadd_256 = _mm256_hadd_ps(s_inst_hadd_256, s_inst_hadd_256);
s_acum += LOOKUP_M256(s_inst_hadd_256, 0);
LOOKUP_M128(e_128, 0) = s_acum - y[i];
s_acum += LOOKUP_M256(s_inst_hadd_256,4);
LOOKUP_M128(e_128, 1) = s_acum - y[i];
s_acum += LOOKUP_M256(s_inst_hadd_256,1);
LOOKUP_M128(e_128, 2) = s_acum - y[i];
s_acum += LOOKUP_M256(s_inst_hadd_256,5);
LOOKUP_M128(e_128, 3) = s_acum - y[i];
s_acum += s_inst_hadd_256[0];
e_128[0] = s_acum - y[i];
s_acum += s_inst_hadd_256[4];
e_128[1] = s_acum - y[i];
s_acum += s_inst_hadd_256[1];
e_128[2] = s_acum - y[i];
s_acum += s_inst_hadd_256[5];
e_128[3] = s_acum - y[i];
__m128 accumulated_error = _mm_load_ps(a_p);
accumulated_error = _mm_fmadd_ps(e_128, e_128, accumulated_error);
@ -219,8 +209,8 @@ void MatchedFilterCore_AVX2(size_t x_start_index,
x2_sum_256 = _mm256_add_ps(x2_sum_256, x2_sum_256_8);
s_256 = _mm256_add_ps(s_256, s_256_8);
__m128 sum = hsum_ab(x2_sum_256, s_256);
x2_sum += LOOKUP_M128(sum, 0);
s += LOOKUP_M128(sum, 1);
x2_sum += sum[0];
s += sum[1];
// Compute the matched filter error.
float e = y[i] - s;

12
webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator.cc
View File

@ -15,7 +15,6 @@
#include "modules/audio_processing/logging/apm_data_dumper.h"
#include "rtc_base/checks.h"
#include "rtc_base/numerics/safe_minmax.h"
#include "system_wrappers/include/field_trial.h"
namespace webrtc {
namespace {
@ -63,8 +62,8 @@ void MatchedFilterLagAggregator::Reset(bool hard_reset) {
}
}
absl::optional<DelayEstimate> MatchedFilterLagAggregator::Aggregate(
const absl::optional<const MatchedFilter::LagEstimate>& lag_estimate) {
std::optional<DelayEstimate> MatchedFilterLagAggregator::Aggregate(
const std::optional<const MatchedFilter::LagEstimate>& lag_estimate) {
if (lag_estimate && pre_echo_lag_aggregator_) {
pre_echo_lag_aggregator_->Dump(data_dumper_);
pre_echo_lag_aggregator_->Aggregate(
@ -91,7 +90,7 @@ absl::optional<DelayEstimate> MatchedFilterLagAggregator::Aggregate(
}
}
return absl::nullopt;
return std::nullopt;
}
MatchedFilterLagAggregator::HighestPeakAggregator::HighestPeakAggregator(
@ -124,8 +123,6 @@ MatchedFilterLagAggregator::PreEchoLagAggregator::PreEchoLagAggregator(
size_t max_filter_lag,
size_t down_sampling_factor)
: block_size_log2_(GetDownSamplingBlockSizeLog2(down_sampling_factor)),
penalize_high_delays_initial_phase_(!field_trial::IsDisabled(
"WebRTC-Aec3PenalyzeHighDelaysInitialPhase")),
histogram_(
((max_filter_lag + 1) * down_sampling_factor) >> kBlockSizeLog2,
0) {
@ -156,8 +153,7 @@ void MatchedFilterLagAggregator::PreEchoLagAggregator::Aggregate(
++histogram_[histogram_data_[histogram_data_index_]];
histogram_data_index_ = (histogram_data_index_ + 1) % histogram_data_.size();
int pre_echo_candidate_block_size = 0;
if (penalize_high_delays_initial_phase_ &&
number_updates_ < kNumBlocksPerSecond * 2) {
if (number_updates_ < kNumBlocksPerSecond * 2) {
number_updates_++;
float penalization_per_delay = 1.0f;
float max_histogram_value = -1.0f;

7
webrtc/modules/audio_processing/aec3/matched_filter_lag_aggregator.h
View File

@ -11,9 +11,9 @@
#ifndef MODULES_AUDIO_PROCESSING_AEC3_MATCHED_FILTER_LAG_AGGREGATOR_H_
#define MODULES_AUDIO_PROCESSING_AEC3_MATCHED_FILTER_LAG_AGGREGATOR_H_
#include <optional>
#include <vector>
#include "absl/types/optional.h"
#include "api/audio/echo_canceller3_config.h"
#include "modules/audio_processing/aec3/delay_estimate.h"
#include "modules/audio_processing/aec3/matched_filter.h"
@ -41,8 +41,8 @@ class MatchedFilterLagAggregator {
void Reset(bool hard_reset);
// Aggregates the provided lag estimates.
absl::optional<DelayEstimate> Aggregate(
const absl::optional<const MatchedFilter::LagEstimate>& lag_estimate);
std::optional<DelayEstimate> Aggregate(
const std::optional<const MatchedFilter::LagEstimate>& lag_estimate);
// Returns whether a reliable delay estimate has been found.
bool ReliableDelayFound() const { return significant_candidate_found_; }
@ -64,7 +64,6 @@ class MatchedFilterLagAggregator {
private:
const int block_size_log2_;
const bool penalize_high_delays_initial_phase_;
std::array<int, 250> histogram_data_;
std::vector<int> histogram_;
int histogram_data_index_ = 0;

6
webrtc/modules/audio_processing/aec3/multi_channel_content_detector.cc
View File

@ -94,9 +94,9 @@ MultiChannelContentDetector::MultiChannelContentDetector(
detection_threshold_(detection_threshold),
detection_timeout_threshold_frames_(
stereo_detection_timeout_threshold_seconds > 0
? absl::make_optional(stereo_detection_timeout_threshold_seconds *
kNumFramesPerSecond)
: absl::nullopt),
? std::make_optional(stereo_detection_timeout_threshold_seconds *
kNumFramesPerSecond)
: std::nullopt),
stereo_detection_hysteresis_frames_(static_cast<int>(
stereo_detection_hysteresis_seconds * kNumFramesPerSecond)),
metrics_logger_((detect_stereo_content && num_render_input_channels > 1)

5
webrtc/modules/audio_processing/aec3/multi_channel_content_detector.h
View File

@ -14,10 +14,9 @@
#include <stddef.h>
#include <memory>
#include <optional>
#include <vector>
#include "absl/types/optional.h"
namespace webrtc {
// Analyzes audio content to determine whether the contained audio is proper
@ -76,7 +75,7 @@ class MultiChannelContentDetector {
const bool detect_stereo_content_;
const float detection_threshold_;
const absl::optional<int> detection_timeout_threshold_frames_;
const std::optional<int> detection_timeout_threshold_frames_;
const int stereo_detection_hysteresis_frames_;
// Collects and reports metrics on the amount of multichannel content

8
webrtc/modules/audio_processing/aec3/render_delay_buffer.cc
View File

@ -17,9 +17,9 @@
#include <cmath>
#include <memory>
#include <numeric>
#include <optional>
#include <vector>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/audio/echo_canceller3_config.h"
#include "modules/audio_processing/aec3/aec3_common.h"
@ -80,7 +80,7 @@ class RenderDelayBufferImpl final : public RenderDelayBuffer {
BlockBuffer blocks_;
SpectrumBuffer spectra_;
FftBuffer ffts_;
absl::optional<size_t> delay_;
std::optional<size_t> delay_;
RenderBuffer echo_remover_buffer_;
DownsampledRenderBuffer low_rate_;
AlignmentMixer render_mixer_;
@ -95,7 +95,7 @@ class RenderDelayBufferImpl final : public RenderDelayBuffer {
int64_t render_call_counter_ = 0;
bool render_activity_ = false;
size_t render_activity_counter_ = 0;
absl::optional<int> external_audio_buffer_delay_;
std::optional<int> external_audio_buffer_delay_;
bool external_audio_buffer_delay_verified_after_reset_ = false;
size_t min_latency_blocks_ = 0;
size_t excess_render_detection_counter_ = 0;
@ -193,7 +193,7 @@ void RenderDelayBufferImpl::Reset() {
ApplyTotalDelay(config_.delay.default_delay);
// Unset the delays which are set by AlignFromDelay.
delay_ = absl::nullopt;
delay_ = std::nullopt;
}
}

25
webrtc/modules/audio_processing/aec3/render_delay_controller.cc
View File

@ -14,8 +14,8 @@
#include <algorithm>
#include <atomic>
#include <memory>
#include <optional>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/audio/echo_canceller3_config.h"
#include "modules/audio_processing/aec3/aec3_common.h"
@ -44,7 +44,7 @@ class RenderDelayControllerImpl final : public RenderDelayController {
~RenderDelayControllerImpl() override;
void Reset(bool reset_delay_confidence) override;
void LogRenderCall() override;
absl::optional<DelayEstimate> GetDelay(
std::optional<DelayEstimate> GetDelay(
const DownsampledRenderBuffer& render_buffer,
size_t render_delay_buffer_delay,
const Block& capture) override;
@ -54,17 +54,17 @@ class RenderDelayControllerImpl final : public RenderDelayController {
static std::atomic<int> instance_count_;
std::unique_ptr<ApmDataDumper> data_dumper_;
const int hysteresis_limit_blocks_;
absl::optional<DelayEstimate> delay_;
std::optional<DelayEstimate> delay_;
EchoPathDelayEstimator delay_estimator_;
RenderDelayControllerMetrics metrics_;
absl::optional<DelayEstimate> delay_samples_;
std::optional<DelayEstimate> delay_samples_;
size_t capture_call_counter_ = 0;
int delay_change_counter_ = 0;
DelayEstimate::Quality last_delay_estimate_quality_;
};
DelayEstimate ComputeBufferDelay(
const absl::optional<DelayEstimate>& current_delay,
const std::optional<DelayEstimate>& current_delay,
int hysteresis_limit_blocks,
DelayEstimate estimated_delay) {
// Compute the buffer delay increase required to achieve the desired latency.
@ -100,8 +100,8 @@ RenderDelayControllerImpl::RenderDelayControllerImpl(
RenderDelayControllerImpl::~RenderDelayControllerImpl() = default;
void RenderDelayControllerImpl::Reset(bool reset_delay_confidence) {
delay_ = absl::nullopt;
delay_samples_ = absl::nullopt;
delay_ = std::nullopt;
delay_samples_ = std::nullopt;
delay_estimator_.Reset(reset_delay_confidence);
delay_change_counter_ = 0;
if (reset_delay_confidence) {
@ -111,7 +111,7 @@ void RenderDelayControllerImpl::Reset(bool reset_delay_confidence) {
void RenderDelayControllerImpl::LogRenderCall() {}
absl::optional<DelayEstimate> RenderDelayControllerImpl::GetDelay(
std::optional<DelayEstimate> RenderDelayControllerImpl::GetDelay(
const DownsampledRenderBuffer& render_buffer,
size_t render_delay_buffer_delay,
const Block& capture) {
@ -155,11 +155,10 @@ absl::optional<DelayEstimate> RenderDelayControllerImpl::GetDelay(
last_delay_estimate_quality_ = delay_samples_->quality;
}
metrics_.Update(
delay_samples_ ? absl::optional<size_t>(delay_samples_->delay)
: absl::nullopt,
delay_ ? absl::optional<size_t>(delay_->delay) : absl::nullopt,
delay_estimator_.Clockdrift());
metrics_.Update(delay_samples_ ? std::optional<size_t>(delay_samples_->delay)
: std::nullopt,
delay_ ? std::optional<size_t>(delay_->delay) : std::nullopt,
delay_estimator_.Clockdrift());
data_dumper_->DumpRaw("aec3_render_delay_controller_delay",
delay_samples ? delay_samples->delay : 0);

5
webrtc/modules/audio_processing/aec3/render_delay_controller.h
View File

@ -11,7 +11,8 @@
#ifndef MODULES_AUDIO_PROCESSING_AEC3_RENDER_DELAY_CONTROLLER_H_
#define MODULES_AUDIO_PROCESSING_AEC3_RENDER_DELAY_CONTROLLER_H_
#include "absl/types/optional.h"
#include <optional>
#include "api/array_view.h"
#include "api/audio/echo_canceller3_config.h"
#include "modules/audio_processing/aec3/block.h"
@ -38,7 +39,7 @@ class RenderDelayController {
virtual void LogRenderCall() = 0;
// Aligns the render buffer content with the capture signal.
virtual absl::optional<DelayEstimate> GetDelay(
virtual std::optional<DelayEstimate> GetDelay(
const DownsampledRenderBuffer& render_buffer,
size_t render_delay_buffer_delay,
const Block& capture) = 0;

4
webrtc/modules/audio_processing/aec3/render_delay_controller_metrics.cc
View File

@ -42,8 +42,8 @@ enum class DelayChangesCategory {
RenderDelayControllerMetrics::RenderDelayControllerMetrics() = default;
void RenderDelayControllerMetrics::Update(
absl::optional<size_t> delay_samples,
absl::optional<size_t> buffer_delay_blocks,
std::optional<size_t> delay_samples,
std::optional<size_t> buffer_delay_blocks,
ClockdriftDetector::Level clockdrift) {
++call_counter_;

7
webrtc/modules/audio_processing/aec3/render_delay_controller_metrics.h
View File

@ -13,7 +13,8 @@
#include <stddef.h>
#include "absl/types/optional.h"
#include <optional>
#include "modules/audio_processing/aec3/clockdrift_detector.h"
namespace webrtc {
@ -28,8 +29,8 @@ class RenderDelayControllerMetrics {
delete;
// Updates the metric with new data.
void Update(absl::optional<size_t> delay_samples,
absl::optional<size_t> buffer_delay_blocks,
void Update(std::optional<size_t> delay_samples,
std::optional<size_t> buffer_delay_blocks,
ClockdriftDetector::Level clockdrift);
private:

8
webrtc/modules/audio_processing/aec3/render_signal_analyzer.cc
View File

@ -27,7 +27,7 @@ constexpr size_t kCounterThreshold = 5;
// Identifies local bands with narrow characteristics.
void IdentifySmallNarrowBandRegions(
const RenderBuffer& render_buffer,
const absl::optional<size_t>& delay_partitions,
const std::optional<size_t>& delay_partitions,
std::array<size_t, kFftLengthBy2 - 1>* narrow_band_counters) {
RTC_DCHECK(narrow_band_counters);
@ -56,14 +56,14 @@ void IdentifySmallNarrowBandRegions(
// Identifies whether the signal has a single strong narrow-band component.
void IdentifyStrongNarrowBandComponent(const RenderBuffer& render_buffer,
int strong_peak_freeze_duration,
absl::optional<int>* narrow_peak_band,
std::optional<int>* narrow_peak_band,
size_t* narrow_peak_counter) {
RTC_DCHECK(narrow_peak_band);
RTC_DCHECK(narrow_peak_counter);
if (*narrow_peak_band &&
++(*narrow_peak_counter) >
static_cast<size_t>(strong_peak_freeze_duration)) {
*narrow_peak_band = absl::nullopt;
*narrow_peak_band = std::nullopt;
}
const Block& x_latest = render_buffer.GetBlock(0);
@ -125,7 +125,7 @@ RenderSignalAnalyzer::~RenderSignalAnalyzer() = default;
void RenderSignalAnalyzer::Update(
const RenderBuffer& render_buffer,
const absl::optional<size_t>& delay_partitions) {
const std::optional<size_t>& delay_partitions) {
// Identify bands of narrow nature.
IdentifySmallNarrowBandRegions(render_buffer, delay_partitions,
&narrow_band_counters_);

8
webrtc/modules/audio_processing/aec3/render_signal_analyzer.h
View File

@ -14,8 +14,8 @@
#include <algorithm>
#include <array>
#include <cstddef>
#include <optional>
#include "absl/types/optional.h"
#include "api/audio/echo_canceller3_config.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/render_buffer.h"
@ -34,7 +34,7 @@ class RenderSignalAnalyzer {
// Updates the render signal analysis with the most recent render signal.
void Update(const RenderBuffer& render_buffer,
const absl::optional<size_t>& delay_partitions);
const std::optional<size_t>& delay_partitions);
// Returns true if the render signal is poorly exciting.
bool PoorSignalExcitation() const {
@ -48,12 +48,12 @@ class RenderSignalAnalyzer {
void MaskRegionsAroundNarrowBands(
std::array<float, kFftLengthBy2Plus1>* v) const;
absl::optional<int> NarrowPeakBand() const { return narrow_peak_band_; }
std::optional<int> NarrowPeakBand() const { return narrow_peak_band_; }
private:
const int strong_peak_freeze_duration_;
std::array<size_t, kFftLengthBy2 - 1> narrow_band_counters_;
absl::optional<int> narrow_peak_band_;
std::optional<int> narrow_peak_band_;
size_t narrow_peak_counter_;
};

2
webrtc/modules/audio_processing/aec3/residual_echo_estimator.h
View File

@ -13,8 +13,8 @@
#include <array>
#include <memory>
#include <optional>
#include "absl/types/optional.h"
#include "api/audio/echo_canceller3_config.h"
#include "modules/audio_processing/aec3/aec3_common.h"
#include "modules/audio_processing/aec3/aec_state.h"

2
webrtc/modules/audio_processing/aec3/reverb_decay_estimator.cc
View File

@ -102,7 +102,7 @@ ReverbDecayEstimator::ReverbDecayEstimator(const EchoCanceller3Config& config)
ReverbDecayEstimator::~ReverbDecayEstimator() = default;
void ReverbDecayEstimator::Update(rtc::ArrayView<const float> filter,
const absl::optional<float>& filter_quality,
const std::optional<float>& filter_quality,
int filter_delay_blocks,
bool usable_linear_filter,
bool stationary_signal) {

4
webrtc/modules/audio_processing/aec3/reverb_decay_estimator.h
View File

@ -12,9 +12,9 @@
#define MODULES_AUDIO_PROCESSING_AEC3_REVERB_DECAY_ESTIMATOR_H_
#include <array>
#include <optional>
#include <vector>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "modules/audio_processing/aec3/aec3_common.h" // kMaxAdaptiveFilter...
@ -30,7 +30,7 @@ class ReverbDecayEstimator {
~ReverbDecayEstimator();
// Updates the decay estimate.
void Update(rtc::ArrayView<const float> filter,
const absl::optional<float>& filter_quality,
const std::optional<float>& filter_quality,
int filter_delay_blocks,
bool usable_linear_filter,
bool stationary_signal);

2
webrtc/modules/audio_processing/aec3/reverb_frequency_response.cc
View File

@ -62,7 +62,7 @@ void ReverbFrequencyResponse::Update(
const std::vector<std::array<float, kFftLengthBy2Plus1>>&
frequency_response,
int filter_delay_blocks,
const absl::optional<float>& linear_filter_quality,
const std::optional<float>& linear_filter_quality,
bool stationary_block) {
if (stationary_block || !linear_filter_quality) {
return;

4
webrtc/modules/audio_processing/aec3/reverb_frequency_response.h
View File

@ -12,9 +12,9 @@
#define MODULES_AUDIO_PROCESSING_AEC3_REVERB_FREQUENCY_RESPONSE_H_
#include <array>
#include <optional>
#include <vector>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "modules/audio_processing/aec3/aec3_common.h"
@ -31,7 +31,7 @@ class ReverbFrequencyResponse {
void Update(const std::vector<std::array<float, kFftLengthBy2Plus1>>&
frequency_response,
int filter_delay_blocks,
const absl::optional<float>& linear_filter_quality,
const std::optional<float>& linear_filter_quality,
bool stationary_block);
// Returns the estimated frequency response for the reverb.

2
webrtc/modules/audio_processing/aec3/reverb_model_estimator.cc
View File

@ -31,7 +31,7 @@ void ReverbModelEstimator::Update(
rtc::ArrayView<const std::vector<float>> impulse_responses,
rtc::ArrayView<const std::vector<std::array<float, kFftLengthBy2Plus1>>>
frequency_responses,
rtc::ArrayView<const absl::optional<float>> linear_filter_qualities,
rtc::ArrayView<const std::optional<float>> linear_filter_qualities,
rtc::ArrayView<const int> filter_delays_blocks,
const std::vector<bool>& usable_linear_estimates,
bool stationary_block) {

4
webrtc/modules/audio_processing/aec3/reverb_model_estimator.h
View File

@ -13,9 +13,9 @@
#include <array>
#include <memory>
#include <optional>
#include <vector>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/audio/echo_canceller3_config.h"
#include "modules/audio_processing/aec3/aec3_common.h" // kFftLengthBy2Plus1
@ -38,7 +38,7 @@ class ReverbModelEstimator {
rtc::ArrayView<const std::vector<float>> impulse_responses,
rtc::ArrayView<const std::vector<std::array<float, kFftLengthBy2Plus1>>>
frequency_responses,
rtc::ArrayView<const absl::optional<float>> linear_filter_qualities,
rtc::ArrayView<const std::optional<float>> linear_filter_qualities,
rtc::ArrayView<const int> filter_delays_blocks,
const std::vector<bool>& usable_linear_estimates,
bool stationary_block);

2
webrtc/modules/audio_processing/aec3/subtractor_output_analyzer.cc
View File

@ -44,7 +44,7 @@ void SubtractorOutputAnalyzer::Update(
bool coarse_filter_converged_strict =
e2_coarse < 0.05f * y2 && y2 > kConvergenceThreshold;
bool coarse_filter_converged_relaxed =
e2_coarse < 0.2f * y2 && y2 > kConvergenceThresholdLowLevel;
e2_coarse < 0.3f * y2 && y2 > kConvergenceThresholdLowLevel;
float min_e2 = std::min(e2_refined, e2_coarse);
bool filter_diverged = min_e2 > 1.5f * y2 && y2 > 30.f * 30.f * kBlockSize;
filters_converged_[ch] =

4
webrtc/modules/audio_processing/aec3/suppression_gain.cc
View File

@ -107,7 +107,7 @@ float SuppressionGain::UpperBandsGain(
rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>> echo_spectrum,
rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
comfort_noise_spectrum,
const absl::optional<int>& narrow_peak_band,
const std::optional<int>& narrow_peak_band,
bool saturated_echo,
const Block& render,
const std::array<float, kFftLengthBy2Plus1>& low_band_gain) const {
@ -394,7 +394,7 @@ void SuppressionGain::GetGain(
low_band_gain);
// Compute the gain for the upper bands.
const absl::optional<int> narrow_peak_band =
const std::optional<int> narrow_peak_band =
render_signal_analyzer.NarrowPeakBand();
*high_bands_gain =

4
webrtc/modules/audio_processing/aec3/suppression_gain.h
View File

@ -14,9 +14,9 @@
#include <array>
#include <atomic>
#include <memory>
#include <optional>
#include <vector>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/audio/echo_canceller3_config.h"
#include "modules/audio_processing/aec3/aec3_common.h"
@ -70,7 +70,7 @@ class SuppressionGain {
rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>> echo_spectrum,
rtc::ArrayView<const std::array<float, kFftLengthBy2Plus1>>
comfort_noise_spectrum,
const absl::optional<int>& narrow_peak_band,
const std::optional<int>& narrow_peak_band,
bool saturated_echo,
const Block& render,
const std::array<float, kFftLengthBy2Plus1>& low_band_gain) const;

5
webrtc/modules/audio_processing/aec3/transparent_mode.cc
View File

@ -19,6 +19,7 @@ namespace {
constexpr size_t kBlocksSinceConvergencedFilterInit = 10000;
constexpr size_t kBlocksSinceConsistentEstimateInit = 10000;
constexpr float kInitialTransparentStateProbability = 0.2f;
bool DeactivateTransparentMode() {
return field_trial::IsEnabled("WebRTC-Aec3TransparentModeKillSwitch");
@ -41,7 +42,7 @@ class TransparentModeImpl : public TransparentMode {
transparency_activated_ = false;
// The estimated probability of being transparent mode.
prob_transparent_state_ = 0.f;
prob_transparent_state_ = kInitialTransparentStateProbability;
}
void Update(int filter_delay_blocks,
@ -118,7 +119,7 @@ class TransparentModeImpl : public TransparentMode {
private:
bool transparency_activated_ = false;
float prob_transparent_state_ = 0.f;
float prob_transparent_state_ = kInitialTransparentStateProbability;
};
// Legacy classifier for toggling transparent mode.

36
webrtc/modules/audio_processing/aec_dump/aec_dump_factory.h
View File

@ -13,34 +13,34 @@
#include <memory>
#include "absl/base/nullability.h"
#include "absl/strings/string_view.h"
#include "api/task_queue/task_queue_base.h"
#include "modules/audio_processing/include/aec_dump.h"
#include "rtc_base/system/file_wrapper.h"
#include "rtc_base/system/rtc_export.h"
namespace rtc {
class TaskQueue;
} // namespace rtc
namespace webrtc {
class RTC_EXPORT AecDumpFactory {
public:
// The `worker_queue` may not be null and must outlive the created
// AecDump instance. `max_log_size_bytes == -1` means the log size
// will be unlimited. `handle` may not be null. The AecDump takes
// responsibility for `handle` and closes it in the destructor. A
// non-null return value indicates that the file has been
// The `worker_queue` must outlive the created AecDump instance.
// `max_log_size_bytes == -1` means the log size will be unlimited.
// The AecDump takes responsibility for `handle` and closes it in the
// destructor. A non-null return value indicates that the file has been
// sucessfully opened.
static std::unique_ptr<AecDump> Create(webrtc::FileWrapper file,
int64_t max_log_size_bytes,
rtc::TaskQueue* worker_queue);
static std::unique_ptr<AecDump> Create(absl::string_view file_name,
int64_t max_log_size_bytes,
rtc::TaskQueue* worker_queue);
static std::unique_ptr<AecDump> Create(FILE* handle,
int64_t max_log_size_bytes,
rtc::TaskQueue* worker_queue);
static absl::Nullable<std::unique_ptr<AecDump>> Create(
FileWrapper file,
int64_t max_log_size_bytes,
absl::Nonnull<TaskQueueBase*> worker_queue);
static absl::Nullable<std::unique_ptr<AecDump>> Create(
absl::string_view file_name,
int64_t max_log_size_bytes,
absl::Nonnull<TaskQueueBase*> worker_queue);
static absl::Nullable<std::unique_ptr<AecDump>> Create(
absl::Nonnull<FILE*> handle,
int64_t max_log_size_bytes,
absl::Nonnull<TaskQueueBase*> worker_queue);
};
} // namespace webrtc

23
webrtc/modules/audio_processing/aec_dump/null_aec_dump_factory.cc
View File

@ -8,27 +8,32 @@
* be found in the AUTHORS file in the root of the source tree.
*/
#include "absl/base/nullability.h"
#include "absl/strings/string_view.h"
#include "api/task_queue/task_queue_base.h"
#include "modules/audio_processing/aec_dump/aec_dump_factory.h"
#include "modules/audio_processing/include/aec_dump.h"
namespace webrtc {
std::unique_ptr<AecDump> AecDumpFactory::Create(webrtc::FileWrapper file,
int64_t max_log_size_bytes,
rtc::TaskQueue* worker_queue) {
absl::Nullable<std::unique_ptr<AecDump>> AecDumpFactory::Create(
FileWrapper file,
int64_t max_log_size_bytes,
absl::Nonnull<TaskQueueBase*> worker_queue) {
return nullptr;
}
std::unique_ptr<AecDump> AecDumpFactory::Create(absl::string_view file_name,
int64_t max_log_size_bytes,
rtc::TaskQueue* worker_queue) {
absl::Nullable<std::unique_ptr<AecDump>> AecDumpFactory::Create(
absl::string_view file_name,
int64_t max_log_size_bytes,
absl::Nonnull<TaskQueueBase*> worker_queue) {
return nullptr;
}
std::unique_ptr<AecDump> AecDumpFactory::Create(FILE* handle,
int64_t max_log_size_bytes,
rtc::TaskQueue* worker_queue) {
absl::Nullable<std::unique_ptr<AecDump>> AecDumpFactory::Create(
absl::Nonnull<FILE*> handle,
int64_t max_log_size_bytes,
absl::Nonnull<TaskQueueBase*> worker_queue) {
return nullptr;
}
} // namespace webrtc

8
webrtc/modules/audio_processing/agc/BUILD.gn
View File

@ -21,11 +21,11 @@ rtc_library("agc") {
deps = [
":gain_control_interface",
":level_estimation",
"..:api",
"..:apm_logging",
"..:audio_buffer",
"..:audio_frame_view",
"../../../api:array_view",
"../../../api/audio:audio_processing",
"../../../common_audio",
"../../../common_audio:common_audio_c",
"../../../rtc_base:checks",
@ -39,7 +39,6 @@ rtc_library("agc") {
"../agc2:input_volume_stats_reporter",
"../vad",
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}
rtc_library("level_estimation") {
@ -117,10 +116,7 @@ if (rtc_include_tests) {
"../../../test:fileutils",
"../../../test:test_support",
"//testing/gtest",
]
absl_deps = [
"//third_party/abseil-cpp/absl/strings",
"//third_party/abseil-cpp/absl/types:optional",
"//third_party/abseil-cpp/absl/strings:string_view",
]
}
}

22
webrtc/modules/audio_processing/agc/agc_manager_direct.cc
View File

@ -69,11 +69,11 @@ using AnalogAgcConfig =
// string. Returns an unspecified value if the field trial is not specified, if
// disabled or if it cannot be parsed. Example:
// 'WebRTC-Audio-2ndAgcMinMicLevelExperiment/Enabled-80' => returns 80.
absl::optional<int> GetMinMicLevelOverride() {
std::optional<int> GetMinMicLevelOverride() {
constexpr char kMinMicLevelFieldTrial[] =
"WebRTC-Audio-2ndAgcMinMicLevelExperiment";
if (!webrtc::field_trial::IsEnabled(kMinMicLevelFieldTrial)) {
return absl::nullopt;
return std::nullopt;
}
const auto field_trial_string =
webrtc::field_trial::FindFullName(kMinMicLevelFieldTrial);
@ -84,7 +84,7 @@ absl::optional<int> GetMinMicLevelOverride() {
} else {
RTC_LOG(LS_WARNING) << "[agc] Invalid parameter for "
<< kMinMicLevelFieldTrial << ", ignored.";
return absl::nullopt;
return std::nullopt;
}
}
@ -189,8 +189,8 @@ void MonoAgc::Initialize() {
}
void MonoAgc::Process(rtc::ArrayView<const int16_t> audio,
absl::optional<int> rms_error_override) {
new_compression_to_set_ = absl::nullopt;
std::optional<int> rms_error_override) {
new_compression_to_set_ = std::nullopt;
if (check_volume_on_next_process_) {
check_volume_on_next_process_ = false;
@ -617,13 +617,13 @@ void AgcManagerDirect::AnalyzePreProcess(const AudioBuffer& audio_buffer) {
}
void AgcManagerDirect::Process(const AudioBuffer& audio_buffer) {
Process(audio_buffer, /*speech_probability=*/absl::nullopt,
/*speech_level_dbfs=*/absl::nullopt);
Process(audio_buffer, /*speech_probability=*/std::nullopt,
/*speech_level_dbfs=*/std::nullopt);
}
void AgcManagerDirect::Process(const AudioBuffer& audio_buffer,
absl::optional<float> speech_probability,
absl::optional<float> speech_level_dbfs) {
std::optional<float> speech_probability,
std::optional<float> speech_level_dbfs) {
AggregateChannelLevels();
const int volume_after_clipping_handling = recommended_input_volume_;
@ -632,7 +632,7 @@ void AgcManagerDirect::Process(const AudioBuffer& audio_buffer,
}
const size_t num_frames_per_band = audio_buffer.num_frames_per_band();
absl::optional<int> rms_error_override = absl::nullopt;
std::optional<int> rms_error_override = std::nullopt;
if (speech_probability.has_value() && speech_level_dbfs.has_value()) {
rms_error_override =
GetSpeechLevelErrorDb(*speech_level_dbfs, *speech_probability);
@ -656,7 +656,7 @@ void AgcManagerDirect::Process(const AudioBuffer& audio_buffer,
}
}
absl::optional<int> AgcManagerDirect::GetDigitalComressionGain() {
std::optional<int> AgcManagerDirect::GetDigitalComressionGain() {
return new_compressions_to_set_[channel_controlling_gain_];
}

22
webrtc/modules/audio_processing/agc/agc_manager_direct.h
View File

@ -13,13 +13,13 @@
#include <atomic>
#include <memory>
#include <optional>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/audio/audio_processing.h"
#include "modules/audio_processing/agc/agc.h"
#include "modules/audio_processing/agc2/clipping_predictor.h"
#include "modules/audio_processing/audio_buffer.h"
#include "modules/audio_processing/include/audio_processing.h"
#include "modules/audio_processing/logging/apm_data_dumper.h"
#include "rtc_base/gtest_prod_util.h"
@ -75,8 +75,8 @@ class AgcManagerDirect final {
// TODO(webrtc:7494): This signature is needed for testing purposes, unify
// the signatures when the clean-up is done.
void Process(const AudioBuffer& audio_buffer,
absl::optional<float> speech_probability,
absl::optional<float> speech_level_dbfs);
std::optional<float> speech_probability,
std::optional<float> speech_level_dbfs);
// Processes `audio_buffer`. Chooses a digital compression gain and the new
// input volume to recommend. Must be called after `AnalyzePreProcess()`.
@ -100,7 +100,7 @@ class AgcManagerDirect final {
// If available, returns the latest digital compression gain that has been
// chosen.
absl::optional<int> GetDigitalComressionGain();
std::optional<int> GetDigitalComressionGain();
// Returns true if clipping prediction is enabled.
bool clipping_predictor_enabled() const { return !!clipping_predictor_; }
@ -150,7 +150,7 @@ class AgcManagerDirect final {
const bool analog_controller_enabled_;
const absl::optional<int> min_mic_level_override_;
const std::optional<int> min_mic_level_override_;
std::unique_ptr<ApmDataDumper> data_dumper_;
static std::atomic<int> instance_counter_;
const int num_capture_channels_;
@ -176,7 +176,7 @@ class AgcManagerDirect final {
const int clipped_wait_frames_;
std::vector<std::unique_ptr<MonoAgc>> channel_agcs_;
std::vector<absl::optional<int>> new_compressions_to_set_;
std::vector<std::optional<int>> new_compressions_to_set_;
const std::unique_ptr<ClippingPredictor> clipping_predictor_;
const bool use_clipping_predictor_step_;
@ -213,16 +213,14 @@ class MonoAgc {
// after `HandleClipping()`. If `rms_error_override` has a value, RMS error
// from AGC is overridden by it.
void Process(rtc::ArrayView<const int16_t> audio,
absl::optional<int> rms_error_override);
std::optional<int> rms_error_override);
// Returns the recommended input volume. Must be called after `Process()`.
int recommended_analog_level() const { return recommended_input_volume_; }
float voice_probability() const { return agc_->voice_probability(); }
void ActivateLogging() { log_to_histograms_ = true; }
absl::optional<int> new_compression() const {
return new_compression_to_set_;
}
std::optional<int> new_compression() const { return new_compression_to_set_; }
// Only used for testing.
void set_agc(Agc* agc) { agc_.reset(agc); }
@ -263,7 +261,7 @@ class MonoAgc {
// recommended input volume.
int recommended_input_volume_ = 0;
absl::optional<int> new_compression_to_set_;
std::optional<int> new_compression_to_set_;
bool log_to_histograms_ = false;
const int clipped_level_min_;

50
webrtc/modules/audio_processing/agc2/BUILD.gn
View File

@ -23,9 +23,9 @@ rtc_library("speech_level_estimator") {
deps = [
":common",
"..:api",
"..:apm_logging",
"../../../api:array_view",
"../../../api/audio:audio_processing",
"../../../rtc_base:checks",
"../../../rtc_base:logging",
"../../../rtc_base:safe_minmax",
@ -48,9 +48,9 @@ rtc_library("adaptive_digital_gain_controller") {
deps = [
":common",
":gain_applier",
"..:api",
"..:apm_logging",
"..:audio_frame_view",
"../../../api/audio:audio_frame_api",
"../../../api/audio:audio_processing",
"../../../common_audio",
"../../../rtc_base:checks",
"../../../rtc_base:logging",
@ -81,8 +81,6 @@ rtc_library("saturation_protector") {
"../../../rtc_base:safe_compare",
"../../../rtc_base:safe_minmax",
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}
rtc_library("biquad_filter") {
@ -112,15 +110,13 @@ rtc_library("clipping_predictor") {
deps = [
":gain_map",
"..:api",
"..:audio_frame_view",
"../../../api/audio:audio_processing",
"../../../common_audio",
"../../../rtc_base:checks",
"../../../rtc_base:logging",
"../../../rtc_base:safe_minmax",
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}
rtc_source_set("common") {
@ -150,6 +146,7 @@ rtc_library("fixed_digital") {
"..:apm_logging",
"..:audio_frame_view",
"../../../api:array_view",
"../../../api/audio:audio_frame_api",
"../../../common_audio",
"../../../rtc_base:checks",
"../../../rtc_base:gtest_prod",
@ -157,8 +154,8 @@ rtc_library("fixed_digital") {
"../../../rtc_base:safe_minmax",
"../../../rtc_base:stringutils",
"../../../system_wrappers:metrics",
"//third_party/abseil-cpp/absl/strings:string_view",
]
absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
}
rtc_library("gain_applier") {
@ -175,7 +172,7 @@ rtc_library("gain_applier") {
deps = [
":common",
"..:audio_frame_view",
"../../../api:array_view",
"../../../api/audio:audio_frame_api",
"../../../rtc_base:safe_minmax",
]
}
@ -209,10 +206,10 @@ rtc_library("input_volume_controller") {
":clipping_predictor",
":gain_map",
":input_volume_stats_reporter",
"..:api",
"..:audio_buffer",
"..:audio_frame_view",
"../../../api:array_view",
"../../../api/audio:audio_processing",
"../../../rtc_base:checks",
"../../../rtc_base:checks",
"../../../rtc_base:gtest_prod",
@ -222,8 +219,6 @@ rtc_library("input_volume_controller") {
"../../../system_wrappers:field_trial",
"../../../system_wrappers:metrics",
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}
rtc_library("noise_level_estimator") {
@ -234,8 +229,7 @@ rtc_library("noise_level_estimator") {
deps = [
":biquad_filter",
"..:apm_logging",
"..:audio_frame_view",
"../../../api:array_view",
"../../../api/audio:audio_frame_api",
"../../../rtc_base:checks",
"../../../system_wrappers",
]
@ -268,8 +262,7 @@ rtc_library("vad_wrapper") {
deps = [
":common",
":cpu_features",
"..:audio_frame_view",
"../../../api:array_view",
"../../../api/audio:audio_frame_api",
"../../../common_audio",
"../../../rtc_base:checks",
"rnn_vad",
@ -303,8 +296,8 @@ rtc_library("speech_level_estimator_unittest") {
deps = [
":common",
":speech_level_estimator",
"..:api",
"..:apm_logging",
"../../../api/audio:audio_processing",
"../../../rtc_base:gunit_helpers",
"../../../test:test_support",
]
@ -320,9 +313,9 @@ rtc_library("adaptive_digital_gain_controller_unittest") {
":adaptive_digital_gain_controller",
":common",
":test_utils",
"..:api",
"..:apm_logging",
"..:audio_frame_view",
"../../../api/audio:audio_processing",
"../../../common_audio",
"../../../rtc_base:gunit_helpers",
"../../../test:test_support",
@ -337,7 +330,7 @@ rtc_library("gain_applier_unittest") {
deps = [
":gain_applier",
":test_utils",
"..:audio_frame_view",
"../../../api/audio:audio_frame_api",
"../../../rtc_base:gunit_helpers",
"../../../test:test_support",
]
@ -391,6 +384,7 @@ rtc_library("fixed_digital_unittests") {
"..:apm_logging",
"..:audio_frame_view",
"../../../api:array_view",
"../../../api/audio:audio_frame_api",
"../../../common_audio",
"../../../rtc_base:checks",
"../../../rtc_base:gunit_helpers",
@ -413,7 +407,6 @@ rtc_library("input_volume_controller_unittests") {
":clipping_predictor",
":gain_map",
":input_volume_controller",
"..:api",
"../../../api:array_view",
"../../../rtc_base:checks",
"../../../rtc_base:random",
@ -426,8 +419,6 @@ rtc_library("input_volume_controller_unittests") {
"../../../test:test_support",
"//testing/gtest",
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}
rtc_library("noise_estimator_unittests") {
@ -439,9 +430,8 @@ rtc_library("noise_estimator_unittests") {
":noise_level_estimator",
":test_utils",
"..:apm_logging",
"..:audio_frame_view",
"../../../api:array_view",
"../../../api:function_view",
"../../../api/audio:audio_frame_api",
"../../../rtc_base:checks",
"../../../rtc_base:gunit_helpers",
]
@ -453,7 +443,7 @@ rtc_library("vad_wrapper_unittests") {
deps = [
":common",
":vad_wrapper",
"..:audio_frame_view",
"../../../api/audio:audio_frame_api",
"../../../rtc_base:checks",
"../../../rtc_base:gunit_helpers",
"../../../rtc_base:safe_compare",
@ -475,6 +465,7 @@ rtc_library("test_utils") {
]
deps = [
"..:audio_frame_view",
"../../../api/audio:audio_frame_api",
"../../../rtc_base:checks",
"../../../rtc_base:random",
]
@ -491,10 +482,7 @@ rtc_library("input_volume_stats_reporter") {
"../../../rtc_base:safe_minmax",
"../../../rtc_base:stringutils",
"../../../system_wrappers:metrics",
]
absl_deps = [
"//third_party/abseil-cpp/absl/strings",
"//third_party/abseil-cpp/absl/types:optional",
"//third_party/abseil-cpp/absl/strings:string_view",
]
}
@ -506,6 +494,6 @@ rtc_library("input_volume_stats_reporter_unittests") {
"../../../rtc_base:stringutils",
"../../../system_wrappers:metrics",
"../../../test:test_support",
"//third_party/abseil-cpp/absl/strings:string_view",
]
absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
}

2
webrtc/modules/audio_processing/agc2/adaptive_digital_gain_controller.cc
View File

@ -124,7 +124,7 @@ AdaptiveDigitalGainController::AdaptiveDigitalGainController(
}
void AdaptiveDigitalGainController::Process(const FrameInfo& info,
AudioFrameView<float> frame) {
DeinterleavedView<float> frame) {
RTC_DCHECK_GE(info.speech_level_dbfs, -150.0f);
RTC_DCHECK_GE(frame.num_channels(), 1);
RTC_DCHECK(

6
webrtc/modules/audio_processing/agc2/adaptive_digital_gain_controller.h
View File

@ -13,9 +13,9 @@
#include <vector>
#include "api/audio/audio_processing.h"
#include "api/audio/audio_view.h"
#include "modules/audio_processing/agc2/gain_applier.h"
#include "modules/audio_processing/include/audio_frame_view.h"
#include "modules/audio_processing/include/audio_processing.h"
namespace webrtc {
@ -46,7 +46,7 @@ class AdaptiveDigitalGainController {
// Analyzes `info`, updates the digital gain and applies it to a 10 ms
// `frame`. Supports any sample rate supported by APM.
void Process(const FrameInfo& info, AudioFrameView<float> frame);
void Process(const FrameInfo& info, DeinterleavedView<float> frame);
private:
ApmDataDumper* const apm_data_dumper_;

38
webrtc/modules/audio_processing/agc2/clipping_predictor.cc
View File

@ -131,11 +131,11 @@ class ClippingEventPredictor : public ClippingPredictor {
// if at least `GetMinFramesProcessed()` frames have been processed since the
// last reset and a clipping event is predicted. `level`, `min_mic_level`, and
// `max_mic_level` are limited to [0, 255] and `default_step` to [1, 255].
absl::optional<int> EstimateClippedLevelStep(int channel,
int level,
int default_step,
int min_mic_level,
int max_mic_level) const {
std::optional<int> EstimateClippedLevelStep(int channel,
int level,
int default_step,
int min_mic_level,
int max_mic_level) const {
RTC_CHECK_GE(channel, 0);
RTC_CHECK_LT(channel, ch_buffers_.size());
RTC_DCHECK_GE(level, 0);
@ -147,7 +147,7 @@ class ClippingEventPredictor : public ClippingPredictor {
RTC_DCHECK_GE(max_mic_level, 0);
RTC_DCHECK_LE(max_mic_level, 255);
if (level <= min_mic_level) {
return absl::nullopt;
return std::nullopt;
}
if (PredictClippingEvent(channel)) {
const int new_level =
@ -157,7 +157,7 @@ class ClippingEventPredictor : public ClippingPredictor {
return step;
}
}
return absl::nullopt;
return std::nullopt;
}
private:
@ -271,11 +271,11 @@ class ClippingPeakPredictor : public ClippingPredictor {
// least `GetMinFramesProcessed()` frames have been processed since the last
// reset and a clipping event is predicted. `level`, `min_mic_level`, and
// `max_mic_level` are limited to [0, 255] and `default_step` to [1, 255].
absl::optional<int> EstimateClippedLevelStep(int channel,
int level,
int default_step,
int min_mic_level,
int max_mic_level) const {
std::optional<int> EstimateClippedLevelStep(int channel,
int level,
int default_step,
int min_mic_level,
int max_mic_level) const {
RTC_DCHECK_GE(channel, 0);
RTC_DCHECK_LT(channel, ch_buffers_.size());
RTC_DCHECK_GE(level, 0);
@ -287,9 +287,9 @@ class ClippingPeakPredictor : public ClippingPredictor {
RTC_DCHECK_GE(max_mic_level, 0);
RTC_DCHECK_LE(max_mic_level, 255);
if (level <= min_mic_level) {
return absl::nullopt;
return std::nullopt;
}
absl::optional<float> estimate_db = EstimatePeakValue(channel);
std::optional<float> estimate_db = EstimatePeakValue(channel);
if (estimate_db.has_value() && estimate_db.value() > clipping_threshold_) {
int step = 0;
if (!adaptive_step_estimation_) {
@ -309,7 +309,7 @@ class ClippingPeakPredictor : public ClippingPredictor {
return level - new_level;
}
}
return absl::nullopt;
return std::nullopt;
}
private:
@ -319,18 +319,18 @@ class ClippingPeakPredictor : public ClippingPredictor {
// Predicts clipping sample peaks based on the processed audio frames.
// Returns the estimated peak value if clipping is predicted. Otherwise
// returns absl::nullopt.
absl::optional<float> EstimatePeakValue(int channel) const {
// returns std::nullopt.
std::optional<float> EstimatePeakValue(int channel) const {
const auto reference_metrics = ch_buffers_[channel]->ComputePartialMetrics(
reference_window_delay_, reference_window_length_);
if (!reference_metrics.has_value()) {
return absl::nullopt;
return std::nullopt;
}
const auto metrics =
ch_buffers_[channel]->ComputePartialMetrics(0, window_length_);
if (!metrics.has_value() ||
!(FloatS16ToDbfs(metrics.value().max) > clipping_threshold_)) {
return absl::nullopt;
return std::nullopt;
}
const float reference_crest_factor =
ComputeCrestFactor(reference_metrics.value());

8
webrtc/modules/audio_processing/agc2/clipping_predictor.h
View File

@ -12,11 +12,11 @@
#define MODULES_AUDIO_PROCESSING_AGC2_CLIPPING_PREDICTOR_H_
#include <memory>
#include <optional>
#include <vector>
#include "absl/types/optional.h"
#include "api/audio/audio_processing.h"
#include "modules/audio_processing/include/audio_frame_view.h"
#include "modules/audio_processing/include/audio_processing.h"
namespace webrtc {
@ -35,12 +35,12 @@ class ClippingPredictor {
// Predicts if clipping is going to occur for the specified `channel` in the
// near-future and, if so, it returns a recommended analog mic level decrease
// step. Returns absl::nullopt if clipping is not predicted.
// step. Returns std::nullopt if clipping is not predicted.
// `level` is the current analog mic level, `default_step` is the amount the
// mic level is lowered by the analog controller with every clipping event and
// `min_mic_level` and `max_mic_level` is the range of allowed analog mic
// levels.
virtual absl::optional<int> EstimateClippedLevelStep(
virtual std::optional<int> EstimateClippedLevelStep(
int channel,
int level,
int default_step,

6
webrtc/modules/audio_processing/agc2/clipping_predictor_level_buffer.cc
View File

@ -50,7 +50,7 @@ void ClippingPredictorLevelBuffer::Push(Level level) {
}
// TODO(bugs.webrtc.org/12774): Optimize partial computation for long buffers.
absl::optional<ClippingPredictorLevelBuffer::Level>
std::optional<ClippingPredictorLevelBuffer::Level>
ClippingPredictorLevelBuffer::ComputePartialMetrics(int delay,
int num_items) const {
RTC_DCHECK_GE(delay, 0);
@ -59,7 +59,7 @@ ClippingPredictorLevelBuffer::ComputePartialMetrics(int delay,
RTC_DCHECK_LE(num_items, Capacity());
RTC_DCHECK_LE(delay + num_items, Capacity());
if (delay + num_items > Size()) {
return absl::nullopt;
return std::nullopt;
}
float sum = 0.0f;
float max = 0.0f;
@ -71,7 +71,7 @@ ClippingPredictorLevelBuffer::ComputePartialMetrics(int delay,
sum += data_[idx].average;
max = std::fmax(data_[idx].max, max);
}
return absl::optional<Level>({sum / static_cast<float>(num_items), max});
return std::optional<Level>({sum / static_cast<float>(num_items), max});
}
} // namespace webrtc

5
webrtc/modules/audio_processing/agc2/clipping_predictor_level_buffer.h
View File

@ -12,10 +12,9 @@
#define MODULES_AUDIO_PROCESSING_AGC2_CLIPPING_PREDICTOR_LEVEL_BUFFER_H_
#include <memory>
#include <optional>
#include <vector>
#include "absl/types/optional.h"
namespace webrtc {
// A circular buffer to store frame-wise `Level` items for clipping prediction.
@ -58,7 +57,7 @@ class ClippingPredictorLevelBuffer {
// from `delay` to `delay` - `num_items` (a delay equal to zero corresponds
// to the most recently pushed item). The value of `delay` is limited to
// [0, N] and `num_items` to [1, M] where N + M is the capacity of the buffer.
absl::optional<Level> ComputePartialMetrics(int delay, int num_items) const;
std::optional<Level> ComputePartialMetrics(int delay, int num_items) const;
private:
int tail_;

24
webrtc/modules/audio_processing/agc2/fixed_digital_level_estimator.cc
View File

@ -14,6 +14,7 @@
#include <cmath>
#include "api/array_view.h"
#include "api/audio/audio_frame.h"
#include "modules/audio_processing/logging/apm_data_dumper.h"
#include "rtc_base/checks.h"
@ -34,14 +35,17 @@ constexpr float kDecayFilterConstant = 0.9971259f;
} // namespace
FixedDigitalLevelEstimator::FixedDigitalLevelEstimator(
int sample_rate_hz,
size_t samples_per_channel,
ApmDataDumper* apm_data_dumper)
: apm_data_dumper_(apm_data_dumper),
filter_state_level_(kInitialFilterStateLevel) {
SetSampleRate(sample_rate_hz);
SetSamplesPerChannel(samples_per_channel);
CheckParameterCombination();
RTC_DCHECK(apm_data_dumper_);
apm_data_dumper_->DumpRaw("agc2_level_estimator_samplerate", sample_rate_hz);
// Convert `samples_per_channel` to sample rate for
// `agc2_level_estimator_samplerate`.
apm_data_dumper_->DumpRaw("agc2_level_estimator_samplerate",
samples_per_channel * kDefaultAudioBuffersPerSec);
}
void FixedDigitalLevelEstimator::CheckParameterCombination() {
@ -52,15 +56,15 @@ void FixedDigitalLevelEstimator::CheckParameterCombination() {
}
std::array<float, kSubFramesInFrame> FixedDigitalLevelEstimator::ComputeLevel(
const AudioFrameView<const float>& float_frame) {
DeinterleavedView<const float> float_frame) {
RTC_DCHECK_GT(float_frame.num_channels(), 0);
RTC_DCHECK_EQ(float_frame.samples_per_channel(), samples_in_frame_);
// Compute max envelope without smoothing.
std::array<float, kSubFramesInFrame> envelope{};
for (int channel_idx = 0; channel_idx < float_frame.num_channels();
for (size_t channel_idx = 0; channel_idx < float_frame.num_channels();
++channel_idx) {
const auto channel = float_frame.channel(channel_idx);
const auto channel = float_frame[channel_idx];
for (int sub_frame = 0; sub_frame < kSubFramesInFrame; ++sub_frame) {
for (int sample_in_sub_frame = 0;
sample_in_sub_frame < samples_in_sub_frame_; ++sample_in_sub_frame) {
@ -95,7 +99,7 @@ std::array<float, kSubFramesInFrame> FixedDigitalLevelEstimator::ComputeLevel(
// Dump data for debug.
RTC_DCHECK(apm_data_dumper_);
const auto channel = float_frame.channel(0);
const auto channel = float_frame[0];
apm_data_dumper_->DumpRaw("agc2_level_estimator_samples",
samples_in_sub_frame_,
&channel[sub_frame * samples_in_sub_frame_]);
@ -106,9 +110,9 @@ std::array<float, kSubFramesInFrame> FixedDigitalLevelEstimator::ComputeLevel(
return envelope;
}
void FixedDigitalLevelEstimator::SetSampleRate(int sample_rate_hz) {
samples_in_frame_ =
rtc::CheckedDivExact(sample_rate_hz * kFrameDurationMs, 1000);
void FixedDigitalLevelEstimator::SetSamplesPerChannel(
size_t samples_per_channel) {
samples_in_frame_ = static_cast<int>(samples_per_channel);
samples_in_sub_frame_ =
rtc::CheckedDivExact(samples_in_frame_, kSubFramesInFrame);
CheckParameterCombination();

18
webrtc/modules/audio_processing/agc2/fixed_digital_level_estimator.h
View File

@ -25,12 +25,16 @@ class ApmDataDumper;
// filtering.
class FixedDigitalLevelEstimator {
public:
// Sample rates are allowed if the number of samples in a frame
// (sample_rate_hz * kFrameDurationMs / 1000) is divisible by
// `samples_per_channel` is expected to be derived from this formula:
// sample_rate_hz * kFrameDurationMs / 1000
// or, for a 10ms duration:
// sample_rate_hz / 100
// I.e. the number of samples for 10ms of the given sample rate. The
// expectation is that samples per channel is divisible by
// kSubFramesInSample. For kFrameDurationMs=10 and
// kSubFramesInSample=20, this means that sample_rate_hz has to be
// divisible by 2000.
FixedDigitalLevelEstimator(int sample_rate_hz,
// kSubFramesInSample=20, this means that the original sample rate has to be
// divisible by 2000 and therefore `samples_per_channel` by 20.
FixedDigitalLevelEstimator(size_t samples_per_channel,
ApmDataDumper* apm_data_dumper);
FixedDigitalLevelEstimator(const FixedDigitalLevelEstimator&) = delete;
@ -42,11 +46,11 @@ class FixedDigitalLevelEstimator {
// ms of audio produces a level estimates in the same scale. The
// level estimate contains kSubFramesInFrame values.
std::array<float, kSubFramesInFrame> ComputeLevel(
const AudioFrameView<const float>& float_frame);
DeinterleavedView<const float> float_frame);
// Rate may be changed at any time (but not concurrently) from the
// value passed to the constructor. The class is not thread safe.
void SetSampleRate(int sample_rate_hz);
void SetSamplesPerChannel(size_t samples_per_channel);
// Resets the level estimator internal state.
void Reset();

26
webrtc/modules/audio_processing/agc2/gain_applier.cc
View File

@ -10,7 +10,7 @@
#include "modules/audio_processing/agc2/gain_applier.h"
#include "api/array_view.h"
#include "api/audio/audio_view.h"
#include "modules/audio_processing/agc2/agc2_common.h"
#include "rtc_base/numerics/safe_minmax.h"
@ -24,9 +24,9 @@ bool GainCloseToOne(float gain_factor) {
gain_factor <= 1.f + 1.f / kMaxFloatS16Value;
}
void ClipSignal(AudioFrameView<float> signal) {
for (int k = 0; k < signal.num_channels(); ++k) {
rtc::ArrayView<float> channel_view = signal.channel(k);
void ClipSignal(DeinterleavedView<float> signal) {
for (size_t k = 0; k < signal.num_channels(); ++k) {
MonoView<float> channel_view = signal[k];
for (auto& sample : channel_view) {
sample = rtc::SafeClamp(sample, kMinFloatS16Value, kMaxFloatS16Value);
}
@ -36,7 +36,7 @@ void ClipSignal(AudioFrameView<float> signal) {
void ApplyGainWithRamping(float last_gain_linear,
float gain_at_end_of_frame_linear,
float inverse_samples_per_channel,
AudioFrameView<float> float_frame) {
DeinterleavedView<float> float_frame) {
// Do not modify the signal.
if (last_gain_linear == gain_at_end_of_frame_linear &&
GainCloseToOne(gain_at_end_of_frame_linear)) {
@ -45,8 +45,8 @@ void ApplyGainWithRamping(float last_gain_linear,
// Gain is constant and different from 1.
if (last_gain_linear == gain_at_end_of_frame_linear) {
for (int k = 0; k < float_frame.num_channels(); ++k) {
rtc::ArrayView<float> channel_view = float_frame.channel(k);
for (size_t k = 0; k < float_frame.num_channels(); ++k) {
MonoView<float> channel_view = float_frame[k];
for (auto& sample : channel_view) {
sample *= gain_at_end_of_frame_linear;
}
@ -57,12 +57,12 @@ void ApplyGainWithRamping(float last_gain_linear,
// The gain changes. We have to change slowly to avoid discontinuities.
const float increment = (gain_at_end_of_frame_linear - last_gain_linear) *
inverse_samples_per_channel;
float gain = last_gain_linear;
for (int i = 0; i < float_frame.samples_per_channel(); ++i) {
for (int ch = 0; ch < float_frame.num_channels(); ++ch) {
float_frame.channel(ch)[i] *= gain;
for (size_t ch = 0; ch < float_frame.num_channels(); ++ch) {
float gain = last_gain_linear;
for (float& sample : float_frame[ch]) {
sample *= gain;
gain += increment;
}
gain += increment;
}
}
@ -73,7 +73,7 @@ GainApplier::GainApplier(bool hard_clip_samples, float initial_gain_factor)
last_gain_factor_(initial_gain_factor),
current_gain_factor_(initial_gain_factor) {}
void GainApplier::ApplyGain(AudioFrameView<float> signal) {
void GainApplier::ApplyGain(DeinterleavedView<float> signal) {
if (static_cast<int>(signal.samples_per_channel()) != samples_per_channel_) {
Initialize(signal.samples_per_channel());
}

8
webrtc/modules/audio_processing/agc2/gain_applier.h
View File

@ -13,6 +13,7 @@
#include <stddef.h>
#include "api/audio/audio_view.h"
#include "modules/audio_processing/include/audio_frame_view.h"
namespace webrtc {
@ -20,10 +21,15 @@ class GainApplier {
public:
GainApplier(bool hard_clip_samples, float initial_gain_factor);
void ApplyGain(AudioFrameView<float> signal);
void ApplyGain(DeinterleavedView<float> signal);
void SetGainFactor(float gain_factor);
float GetGainFactor() const { return current_gain_factor_; }
[[deprecated("Use DeinterleavedView<> version")]] void ApplyGain(
AudioFrameView<float> signal) {
ApplyGain(signal.view());
}
private:
void Initialize(int samples_per_channel);

14
webrtc/modules/audio_processing/agc2/input_volume_controller.cc
View File

@ -173,7 +173,7 @@ void MonoInputVolumeController::Initialize() {
// previous update and the ratio of non-silence frames (i.e., frames with a
// `speech_probability` higher than `speech_probability_threshold_`) is at least
// `speech_ratio_threshold_`.
void MonoInputVolumeController::Process(absl::optional<int> rms_error_db,
void MonoInputVolumeController::Process(std::optional<int> rms_error_db,
float speech_probability) {
if (check_volume_on_next_process_) {
check_volume_on_next_process_ = false;
@ -404,7 +404,7 @@ void InputVolumeController::Initialize() {
clipping_rate_log_ = 0.0f;
clipping_rate_log_counter_ = 0;
applied_input_volume_ = absl::nullopt;
applied_input_volume_ = std::nullopt;
}
void InputVolumeController::AnalyzeInputAudio(int applied_input_volume,
@ -498,13 +498,13 @@ void InputVolumeController::AnalyzeInputAudio(int applied_input_volume,
AggregateChannelLevels();
}
absl::optional<int> InputVolumeController::RecommendInputVolume(
std::optional<int> InputVolumeController::RecommendInputVolume(
float speech_probability,
absl::optional<float> speech_level_dbfs) {
std::optional<float> speech_level_dbfs) {
// Only process if applied input volume is set.
if (!applied_input_volume_.has_value()) {
RTC_LOG(LS_ERROR) << "[AGC2] Applied input volume not set.";
return absl::nullopt;
return std::nullopt;
}
AggregateChannelLevels();
@ -514,7 +514,7 @@ absl::optional<int> InputVolumeController::RecommendInputVolume(
return applied_input_volume_;
}
absl::optional<int> rms_error_db;
std::optional<int> rms_error_db;
if (speech_level_dbfs.has_value()) {
// Compute the error for all frames (both speech and non-speech frames).
rms_error_db = GetSpeechLevelRmsErrorDb(
@ -533,7 +533,7 @@ absl::optional<int> InputVolumeController::RecommendInputVolume(
recommended_input_volume_);
}
applied_input_volume_ = absl::nullopt;
applied_input_volume_ = std::nullopt;
return recommended_input_volume();
}

14
webrtc/modules/audio_processing/agc2/input_volume_controller.h
View File

@ -12,13 +12,13 @@
#define MODULES_AUDIO_PROCESSING_AGC2_INPUT_VOLUME_CONTROLLER_H_
#include <memory>
#include <optional>
#include <vector>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/audio/audio_processing.h"
#include "modules/audio_processing/agc2/clipping_predictor.h"
#include "modules/audio_processing/audio_buffer.h"
#include "modules/audio_processing/include/audio_processing.h"
#include "rtc_base/gtest_prod_util.h"
namespace webrtc {
@ -50,7 +50,7 @@ class InputVolumeController final {
// Limited to values higher than 0.
int clipped_wait_frames = 300;
// Enables clipping prediction functionality.
bool enable_clipping_predictor = false;
bool enable_clipping_predictor = true;
// Speech level target range (dBFS). If the speech level is in the range
// [`target_range_min_dbfs`, `target_range_max_dbfs`], no input volume
// adjustments are done based on the speech level. For speech levels below
@ -95,9 +95,9 @@ class InputVolumeController final {
// suppression are applied. Returns a non-empty input volume recommendation if
// available. If `capture_output_used_` is true, returns the applied input
// volume.
absl::optional<int> RecommendInputVolume(
std::optional<int> RecommendInputVolume(
float speech_probability,
absl::optional<float> speech_level_dbfs);
std::optional<float> speech_level_dbfs);
// Stores whether the capture output will be used or not. Call when the
// capture stream output has been flagged to be used/not-used. If unused, the
@ -155,7 +155,7 @@ class InputVolumeController final {
int recommended_input_volume_ = 0;
// Applied input volume. After `SetAppliedInputVolume()` is called it holds
// the current applied volume.
absl::optional<int> applied_input_volume_;
std::optional<int> applied_input_volume_;
bool capture_output_used_;
@ -213,7 +213,7 @@ class MonoInputVolumeController {
// result of `HandleClipping()` and on `rms_error_dbfs`. Updates are only
// allowed for active speech segments and when `rms_error_dbfs` is not empty.
// Must be called after `HandleClipping()`.
void Process(absl::optional<int> rms_error_dbfs, float speech_probability);
void Process(std::optional<int> rms_error_dbfs, float speech_probability);
// Returns the recommended input volume. Must be called after `Process()`.
int recommended_analog_level() const { return recommended_input_volume_; }

5
webrtc/modules/audio_processing/agc2/input_volume_stats_reporter.h
View File

@ -11,7 +11,8 @@
#ifndef MODULES_AUDIO_PROCESSING_AGC2_INPUT_VOLUME_STATS_REPORTER_H_
#define MODULES_AUDIO_PROCESSING_AGC2_INPUT_VOLUME_STATS_REPORTER_H_
#include "absl/types/optional.h"
#include <optional>
#include "rtc_base/gtest_prod_util.h"
#include "system_wrappers/include/metrics.h"
@ -83,7 +84,7 @@ class InputVolumeStatsReporter {
const bool cannot_log_stats_;
int log_volume_update_stats_counter_ = 0;
absl::optional<int> previous_input_volume_ = absl::nullopt;
std::optional<int> previous_input_volume_ = std::nullopt;
};
// Updates the histogram that keeps track of recommended input volume changes

61
webrtc/modules/audio_processing/agc2/limiter.cc
View File

@ -46,22 +46,20 @@ void InterpolateFirstSubframe(float last_factor,
void ComputePerSampleSubframeFactors(
const std::array<float, kSubFramesInFrame + 1>& scaling_factors,
int samples_per_channel,
rtc::ArrayView<float> per_sample_scaling_factors) {
const int num_subframes = scaling_factors.size() - 1;
const int subframe_size =
rtc::CheckedDivExact(samples_per_channel, num_subframes);
MonoView<float> per_sample_scaling_factors) {
const size_t num_subframes = scaling_factors.size() - 1;
const int subframe_size = rtc::CheckedDivExact(
SamplesPerChannel(per_sample_scaling_factors), num_subframes);
// Handle first sub-frame differently in case of attack.
const bool is_attack = scaling_factors[0] > scaling_factors[1];
if (is_attack) {
InterpolateFirstSubframe(
scaling_factors[0], scaling_factors[1],
rtc::ArrayView<float>(
per_sample_scaling_factors.subview(0, subframe_size)));
per_sample_scaling_factors.subview(0, subframe_size));
}
for (int i = is_attack ? 1 : 0; i < num_subframes; ++i) {
for (size_t i = is_attack ? 1 : 0; i < num_subframes; ++i) {
const int subframe_start = i * subframe_size;
const float scaling_start = scaling_factors[i];
const float scaling_end = scaling_factors[i + 1];
@ -73,39 +71,36 @@ void ComputePerSampleSubframeFactors(
}
}
void ScaleSamples(rtc::ArrayView<const float> per_sample_scaling_factors,
AudioFrameView<float> signal) {
void ScaleSamples(MonoView<const float> per_sample_scaling_factors,
DeinterleavedView<float> signal) {
const int samples_per_channel = signal.samples_per_channel();
RTC_DCHECK_EQ(samples_per_channel, per_sample_scaling_factors.size());
for (int i = 0; i < signal.num_channels(); ++i) {
rtc::ArrayView<float> channel = signal.channel(i);
RTC_DCHECK_EQ(samples_per_channel,
SamplesPerChannel(per_sample_scaling_factors));
for (size_t i = 0; i < signal.num_channels(); ++i) {
MonoView<float> channel = signal[i];
for (int j = 0; j < samples_per_channel; ++j) {
channel[j] = rtc::SafeClamp(channel[j] * per_sample_scaling_factors[j],
kMinFloatS16Value, kMaxFloatS16Value);
}
}
}
void CheckLimiterSampleRate(int sample_rate_hz) {
// Check that per_sample_scaling_factors_ is large enough.
RTC_DCHECK_LE(sample_rate_hz,
kMaximalNumberOfSamplesPerChannel * 1000 / kFrameDurationMs);
}
} // namespace
Limiter::Limiter(int sample_rate_hz,
ApmDataDumper* apm_data_dumper,
Limiter::Limiter(ApmDataDumper* apm_data_dumper,
size_t samples_per_channel,
absl::string_view histogram_name)
: interp_gain_curve_(apm_data_dumper, histogram_name),
level_estimator_(sample_rate_hz, apm_data_dumper),
level_estimator_(samples_per_channel, apm_data_dumper),
apm_data_dumper_(apm_data_dumper) {
CheckLimiterSampleRate(sample_rate_hz);
RTC_DCHECK_LE(samples_per_channel, kMaximalNumberOfSamplesPerChannel);
}
Limiter::~Limiter() = default;
void Limiter::Process(AudioFrameView<float> signal) {
void Limiter::Process(DeinterleavedView<float> signal) {
RTC_DCHECK_LE(signal.samples_per_channel(),
kMaximalNumberOfSamplesPerChannel);
const std::array<float, kSubFramesInFrame> level_estimate =
level_estimator_.ComputeLevel(signal);
@ -116,13 +111,9 @@ void Limiter::Process(AudioFrameView<float> signal) {
return interp_gain_curve_.LookUpGainToApply(x);
});
const int samples_per_channel = signal.samples_per_channel();
RTC_DCHECK_LE(samples_per_channel, kMaximalNumberOfSamplesPerChannel);
auto per_sample_scaling_factors = rtc::ArrayView<float>(
&per_sample_scaling_factors_[0], samples_per_channel);
ComputePerSampleSubframeFactors(scaling_factors_, samples_per_channel,
per_sample_scaling_factors);
MonoView<float> per_sample_scaling_factors(&per_sample_scaling_factors_[0],
signal.samples_per_channel());
ComputePerSampleSubframeFactors(scaling_factors_, per_sample_scaling_factors);
ScaleSamples(per_sample_scaling_factors, signal);
last_scaling_factor_ = scaling_factors_.back();
@ -139,9 +130,9 @@ InterpolatedGainCurve::Stats Limiter::GetGainCurveStats() const {
return interp_gain_curve_.get_stats();
}
void Limiter::SetSampleRate(int sample_rate_hz) {
CheckLimiterSampleRate(sample_rate_hz);
level_estimator_.SetSampleRate(sample_rate_hz);
void Limiter::SetSamplesPerChannel(size_t samples_per_channel) {
RTC_DCHECK_LE(samples_per_channel, kMaximalNumberOfSamplesPerChannel);
level_estimator_.SetSamplesPerChannel(samples_per_channel);
}
void Limiter::Reset() {

21
webrtc/modules/audio_processing/agc2/limiter.h
View File

@ -14,6 +14,7 @@
#include <vector>
#include "absl/strings/string_view.h"
#include "api/audio/audio_frame.h"
#include "modules/audio_processing/agc2/fixed_digital_level_estimator.h"
#include "modules/audio_processing/agc2/interpolated_gain_curve.h"
#include "modules/audio_processing/include/audio_frame_view.h"
@ -23,23 +24,25 @@ class ApmDataDumper;
class Limiter {
public:
Limiter(int sample_rate_hz,
ApmDataDumper* apm_data_dumper,
// See `SetSamplesPerChannel()` for valid values for `samples_per_channel`.
Limiter(ApmDataDumper* apm_data_dumper,
size_t samples_per_channel,
absl::string_view histogram_name_prefix);
Limiter(const Limiter& limiter) = delete;
Limiter& operator=(const Limiter& limiter) = delete;
~Limiter();
// Applies limiter and hard-clipping to `signal`.
void Process(AudioFrameView<float> signal);
void Process(DeinterleavedView<float> signal);
InterpolatedGainCurve::Stats GetGainCurveStats() const;
// Supported rates must be
// * supported by FixedDigitalLevelEstimator
// * below kMaximalNumberOfSamplesPerChannel*1000/kFrameDurationMs
// so that samples_per_channel fit in the
// per_sample_scaling_factors_ array.
void SetSampleRate(int sample_rate_hz);
// Supported values must be
// * Supported by FixedDigitalLevelEstimator
// * Below or equal to kMaximalNumberOfSamplesPerChannel so that samples
// fit in the per_sample_scaling_factors_ array.
void SetSamplesPerChannel(size_t samples_per_channel);
// Resets the internal state.
void Reset();

11
webrtc/modules/audio_processing/agc2/noise_level_estimator.cc
View File

@ -16,7 +16,7 @@
#include <cmath>
#include <numeric>
#include "api/array_view.h"
#include "api/audio/audio_view.h"
#include "modules/audio_processing/logging/apm_data_dumper.h"
#include "rtc_base/checks.h"
@ -25,11 +25,12 @@ namespace {
constexpr int kFramesPerSecond = 100;
float FrameEnergy(const AudioFrameView<const float>& audio) {
float FrameEnergy(DeinterleavedView<const float> audio) {
float energy = 0.0f;
for (int k = 0; k < audio.num_channels(); ++k) {
for (size_t k = 0; k < audio.num_channels(); ++k) {
MonoView<const float> ch = audio[k];
float channel_energy =
std::accumulate(audio.channel(k).begin(), audio.channel(k).end(), 0.0f,
std::accumulate(ch.begin(), ch.end(), 0.0f,
[](float a, float b) -> float { return a + b * b; });
energy = std::max(channel_energy, energy);
}
@ -81,7 +82,7 @@ class NoiseFloorEstimator : public NoiseLevelEstimator {
NoiseFloorEstimator& operator=(const NoiseFloorEstimator&) = delete;
~NoiseFloorEstimator() = default;
float Analyze(const AudioFrameView<const float>& frame) override {
float Analyze(DeinterleavedView<const float> frame) override {
// Detect sample rate changes.
const int sample_rate_hz =
static_cast<int>(frame.samples_per_channel() * kFramesPerSecond);

4
webrtc/modules/audio_processing/agc2/noise_level_estimator.h
View File

@ -13,7 +13,7 @@
#include <memory>
#include "modules/audio_processing/include/audio_frame_view.h"
#include "api/audio/audio_view.h"
namespace webrtc {
class ApmDataDumper;
@ -24,7 +24,7 @@ class NoiseLevelEstimator {
virtual ~NoiseLevelEstimator() = default;
// Analyzes a 10 ms `frame`, updates the noise level estimation and returns
// the value for the latter in dBFS.
virtual float Analyze(const AudioFrameView<const float>& frame) = 0;
virtual float Analyze(DeinterleavedView<const float> frame) = 0;
};
// Creates a noise level estimator based on noise floor detection.

5
webrtc/modules/audio_processing/agc2/rnn_vad/BUILD.gn
View File

@ -100,12 +100,12 @@ rtc_source_set("rnn_vad_layers") {
"../../../../api:function_view",
"../../../../rtc_base:checks",
"../../../../rtc_base:safe_conversions",
"//third_party/abseil-cpp/absl/strings:string_view",
"//third_party/rnnoise:rnn_vad",
]
if (current_cpu == "x86" || current_cpu == "x64") {
deps += [ ":vector_math_avx2" ]
}
absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
}
rtc_source_set("vector_math") {
@ -229,8 +229,8 @@ if (rtc_include_tests) {
"../../../../rtc_base:safe_compare",
"../../../../test:fileutils",
"../../../../test:test_support",
"//third_party/abseil-cpp/absl/strings:string_view",
]
absl_deps = [ "//third_party/abseil-cpp/absl/strings" ]
}
unittest_resources = [
@ -306,7 +306,6 @@ if (rtc_include_tests) {
if (current_cpu == "x86" || current_cpu == "x64") {
deps += [ ":vector_math_avx2" ]
}
absl_deps = [ "//third_party/abseil-cpp/absl/memory" ]
data = unittest_resources
if (is_ios) {
deps += [ ":unittests_bundle_data" ]

4
webrtc/modules/audio_processing/agc2/saturation_protector_buffer.cc
View File

@ -62,9 +62,9 @@ void SaturationProtectorBuffer::PushBack(float v) {
}
}
absl::optional<float> SaturationProtectorBuffer::Front() const {
std::optional<float> SaturationProtectorBuffer::Front() const {
if (size_ == 0) {
return absl::nullopt;
return std::nullopt;
}
RTC_DCHECK_LT(FrontIndex(), buffer_.size());
return buffer_[FrontIndex()];

4
webrtc/modules/audio_processing/agc2/saturation_protector_buffer.h
View File

@ -12,8 +12,8 @@
#define MODULES_AUDIO_PROCESSING_AGC2_SATURATION_PROTECTOR_BUFFER_H_
#include <array>
#include <optional>
#include "absl/types/optional.h"
#include "modules/audio_processing/agc2/agc2_common.h"
namespace webrtc {
@ -43,7 +43,7 @@ class SaturationProtectorBuffer {
// Returns the oldest item in the buffer. Returns an empty value if the
// buffer is empty.
absl::optional<float> Front() const;
std::optional<float> Front() const;
private:
int FrontIndex() const;

2
webrtc/modules/audio_processing/agc2/speech_level_estimator.h
View File

@ -15,8 +15,8 @@
#include <type_traits>
#include "api/audio/audio_processing.h"
#include "modules/audio_processing/agc2/agc2_common.h"
#include "modules/audio_processing/include/audio_processing.h"
namespace webrtc {
class ApmDataDumper;

35
webrtc/modules/audio_processing/agc2/vad_wrapper.cc
View File

@ -13,7 +13,6 @@
#include <array>
#include <utility>
#include "api/array_view.h"
#include "common_audio/resampler/include/push_resampler.h"
#include "modules/audio_processing/agc2/agc2_common.h"
#include "modules/audio_processing/agc2/rnn_vad/common.h"
@ -36,7 +35,7 @@ class MonoVadImpl : public VoiceActivityDetectorWrapper::MonoVad {
int SampleRateHz() const override { return rnn_vad::kSampleRate24kHz; }
void Reset() override { rnn_vad_.Reset(); }
float Analyze(rtc::ArrayView<const float> frame) override {
float Analyze(MonoView<const float> frame) override {
RTC_DCHECK_EQ(frame.size(), rnn_vad::kFrameSize10ms24kHz);
std::array<float, rnn_vad::kFeatureVectorSize> feature_vector;
const bool is_silence = features_extractor_.CheckSilenceComputeFeatures(
@ -73,39 +72,33 @@ VoiceActivityDetectorWrapper::VoiceActivityDetectorWrapper(
int sample_rate_hz)
: vad_reset_period_frames_(
rtc::CheckedDivExact(vad_reset_period_ms, kFrameDurationMs)),
frame_size_(rtc::CheckedDivExact(sample_rate_hz, kNumFramesPerSecond)),
time_to_vad_reset_(vad_reset_period_frames_),
vad_(std::move(vad)) {
RTC_DCHECK(vad_);
vad_(std::move(vad)),
resampled_buffer_(
rtc::CheckedDivExact(vad_->SampleRateHz(), kNumFramesPerSecond)),
resampler_(frame_size_,
resampled_buffer_.size(),
/*num_channels=*/1) {
RTC_DCHECK_GT(vad_reset_period_frames_, 1);
resampled_buffer_.resize(
rtc::CheckedDivExact(vad_->SampleRateHz(), kNumFramesPerSecond));
Initialize(sample_rate_hz);
vad_->Reset();
}
VoiceActivityDetectorWrapper::~VoiceActivityDetectorWrapper() = default;
void VoiceActivityDetectorWrapper::Initialize(int sample_rate_hz) {
RTC_DCHECK_GT(sample_rate_hz, 0);
frame_size_ = rtc::CheckedDivExact(sample_rate_hz, kNumFramesPerSecond);
int status =
resampler_.InitializeIfNeeded(sample_rate_hz, vad_->SampleRateHz(),
/*num_channels=*/1);
constexpr int kStatusOk = 0;
RTC_DCHECK_EQ(status, kStatusOk);
vad_->Reset();
}
float VoiceActivityDetectorWrapper::Analyze(AudioFrameView<const float> frame) {
float VoiceActivityDetectorWrapper::Analyze(
DeinterleavedView<const float> frame) {
// Periodically reset the VAD.
time_to_vad_reset_--;
if (time_to_vad_reset_ <= 0) {
vad_->Reset();
time_to_vad_reset_ = vad_reset_period_frames_;
}
// Resample the first channel of `frame`.
RTC_DCHECK_EQ(frame.samples_per_channel(), frame_size_);
resampler_.Resample(frame.channel(0).data(), frame_size_,
resampled_buffer_.data(), resampled_buffer_.size());
MonoView<float> dst(resampled_buffer_.data(), resampled_buffer_.size());
resampler_.Resample(frame[0], dst);
return vad_->Analyze(resampled_buffer_);
}

14
webrtc/modules/audio_processing/agc2/vad_wrapper.h
View File

@ -14,10 +14,9 @@
#include <memory>
#include <vector>
#include "api/array_view.h"
#include "api/audio/audio_view.h"
#include "common_audio/resampler/include/push_resampler.h"
#include "modules/audio_processing/agc2/cpu_features.h"
#include "modules/audio_processing/include/audio_frame_view.h"
namespace webrtc {
@ -37,7 +36,7 @@ class VoiceActivityDetectorWrapper {
// Resets the internal state.
virtual void Reset() = 0;
// Analyzes an audio frame and returns the speech probability.
virtual float Analyze(rtc::ArrayView<const float> frame) = 0;
virtual float Analyze(MonoView<const float> frame) = 0;
};
// Ctor. Uses `cpu_features` to instantiate the default VAD.
@ -60,21 +59,18 @@ class VoiceActivityDetectorWrapper {
delete;
~VoiceActivityDetectorWrapper();
// Initializes the VAD wrapper.
void Initialize(int sample_rate_hz);
// Analyzes the first channel of `frame` and returns the speech probability.
// `frame` must be a 10 ms frame with the sample rate specified in the last
// `Initialize()` call.
float Analyze(AudioFrameView<const float> frame);
float Analyze(DeinterleavedView<const float> frame);
private:
const int vad_reset_period_frames_;
int frame_size_;
const int frame_size_;
int time_to_vad_reset_;
PushResampler<float> resampler_;
std::unique_ptr<MonoVad> vad_;
std::vector<float> resampled_buffer_;
PushResampler<float> resampler_;
};
} // namespace webrtc

28
webrtc/modules/audio_processing/agc2/vector_float_frame.cc
View File

@ -12,28 +12,20 @@
namespace webrtc {
namespace {
std::vector<float*> ConstructChannelPointers(
std::vector<std::vector<float>>* x) {
std::vector<float*> channel_ptrs;
for (auto& v : *x) {
channel_ptrs.push_back(v.data());
}
return channel_ptrs;
}
} // namespace
VectorFloatFrame::VectorFloatFrame(int num_channels,
int samples_per_channel,
float start_value)
: channels_(num_channels,
std::vector<float>(samples_per_channel, start_value)),
channel_ptrs_(ConstructChannelPointers(&channels_)),
float_frame_view_(channel_ptrs_.data(),
channels_.size(),
samples_per_channel) {}
: channels_(num_channels * samples_per_channel, start_value),
view_(channels_.data(), samples_per_channel, num_channels) {}
VectorFloatFrame::~VectorFloatFrame() = default;
AudioFrameView<float> VectorFloatFrame::float_frame_view() {
return AudioFrameView<float>(view_);
}
AudioFrameView<const float> VectorFloatFrame::float_frame_view() const {
return AudioFrameView<const float>(view_);
}
} // namespace webrtc

17
webrtc/modules/audio_processing/agc2/vector_float_frame.h
View File

@ -13,6 +13,7 @@
#include <vector>
#include "api/audio/audio_view.h"
#include "modules/audio_processing/include/audio_frame_view.h"
namespace webrtc {
@ -24,17 +25,17 @@ class VectorFloatFrame {
VectorFloatFrame(int num_channels,
int samples_per_channel,
float start_value);
const AudioFrameView<float>& float_frame_view() { return float_frame_view_; }
AudioFrameView<const float> float_frame_view() const {
return float_frame_view_;
}
~VectorFloatFrame();
AudioFrameView<float> float_frame_view();
AudioFrameView<const float> float_frame_view() const;
DeinterleavedView<float> view() { return view_; }
DeinterleavedView<const float> view() const { return view_; }
private:
std::vector<std::vector<float>> channels_;
std::vector<float*> channel_ptrs_;
AudioFrameView<float> float_frame_view_;
std::vector<float> channels_;
DeinterleavedView<float> view_;
};
} // namespace webrtc

16
webrtc/modules/audio_processing/audio_buffer.cc
View File

@ -15,7 +15,6 @@
#include <cstdint>
#include "common_audio/channel_buffer.h"
#include "common_audio/include/audio_util.h"
#include "common_audio/resampler/push_sinc_resampler.h"
#include "modules/audio_processing/splitting_filter.h"
#include "rtc_base/checks.h"
@ -25,7 +24,6 @@ namespace {
constexpr size_t kSamplesPer32kHzChannel = 320;
constexpr size_t kSamplesPer48kHzChannel = 480;
constexpr size_t kMaxSamplesPerChannel = AudioBuffer::kMaxSampleRate / 100;
size_t NumBandsFromFramesPerChannel(size_t num_frames) {
if (num_frames == kSamplesPer32kHzChannel) {
@ -110,9 +108,9 @@ void AudioBuffer::CopyFrom(const float* const* stacked_data,
const bool resampling_needed = input_num_frames_ != buffer_num_frames_;
if (downmix_needed) {
RTC_DCHECK_GE(kMaxSamplesPerChannel, input_num_frames_);
RTC_DCHECK_GE(kMaxSamplesPerChannel10ms, input_num_frames_);
std::array<float, kMaxSamplesPerChannel> downmix;
std::array<float, kMaxSamplesPerChannel10ms> downmix;
if (downmix_by_averaging_) {
const float kOneByNumChannels = 1.f / input_num_channels_;
for (size_t i = 0; i < input_num_frames_; ++i) {
@ -230,7 +228,7 @@ void AudioBuffer::CopyFrom(const int16_t* const interleaved_data,
if (num_channels_ == 1) {
if (input_num_channels_ == 1) {
if (resampling_required) {
std::array<float, kMaxSamplesPerChannel> float_buffer;
std::array<float, kMaxSamplesPerChannel10ms> float_buffer;
S16ToFloatS16(interleaved, input_num_frames_, float_buffer.data());
input_resamplers_[0]->Resample(float_buffer.data(), input_num_frames_,
data_->channels()[0],
@ -239,7 +237,7 @@ void AudioBuffer::CopyFrom(const int16_t* const interleaved_data,
S16ToFloatS16(interleaved, input_num_frames_, data_->channels()[0]);
}
} else {
std::array<float, kMaxSamplesPerChannel> float_buffer;
std::array<float, kMaxSamplesPerChannel10ms> float_buffer;
float* downmixed_data =
resampling_required ? float_buffer.data() : data_->channels()[0];
if (downmix_by_averaging_) {
@ -274,7 +272,7 @@ void AudioBuffer::CopyFrom(const int16_t* const interleaved_data,
};
if (resampling_required) {
std::array<float, kMaxSamplesPerChannel> float_buffer;
std::array<float, kMaxSamplesPerChannel10ms> float_buffer;
for (size_t i = 0; i < num_channels_; ++i) {
deinterleave_channel(i, num_channels_, input_num_frames_, interleaved,
float_buffer.data());
@ -302,7 +300,7 @@ void AudioBuffer::CopyTo(const StreamConfig& stream_config,
int16_t* interleaved = interleaved_data;
if (num_channels_ == 1) {
std::array<float, kMaxSamplesPerChannel> float_buffer;
std::array<float, kMaxSamplesPerChannel10ms> float_buffer;
if (resampling_required) {
output_resamplers_[0]->Resample(data_->channels()[0], buffer_num_frames_,
@ -335,7 +333,7 @@ void AudioBuffer::CopyTo(const StreamConfig& stream_config,
if (resampling_required) {
for (size_t i = 0; i < num_channels_; ++i) {
std::array<float, kMaxSamplesPerChannel> float_buffer;
std::array<float, kMaxSamplesPerChannel10ms> float_buffer;
output_resamplers_[i]->Resample(data_->channels()[i],
buffer_num_frames_, float_buffer.data(),
output_num_frames_);

14
webrtc/modules/audio_processing/audio_buffer.h
View File

@ -17,8 +17,10 @@
#include <memory>
#include <vector>
#include "api/audio/audio_processing.h"
#include "api/audio/audio_view.h"
#include "common_audio/channel_buffer.h"
#include "modules/audio_processing/include/audio_processing.h"
#include "common_audio/include/audio_util.h"
namespace webrtc {
@ -32,7 +34,8 @@ enum Band { kBand0To8kHz = 0, kBand8To16kHz = 1, kBand16To24kHz = 2 };
class AudioBuffer {
public:
static const int kSplitBandSize = 160;
static const int kMaxSampleRate = 384000;
// TODO(tommi): Remove this (`AudioBuffer::kMaxSampleRate`) constant.
static const int kMaxSampleRate = webrtc::kMaxSampleRateHz;
AudioBuffer(size_t input_rate,
size_t input_num_channels,
size_t buffer_rate,
@ -56,6 +59,13 @@ class AudioBuffer {
// reset at each call to CopyFrom or InterleaveFrom.
void set_num_channels(size_t num_channels);
// Returns a DeinterleavedView<> over the channel data.
DeinterleavedView<float> view() {
return DeinterleavedView<float>(
num_channels_ && buffer_num_frames_ ? channels()[0] : nullptr,
buffer_num_frames_, num_channels_);
}
size_t num_channels() const { return num_channels_; }
size_t num_frames() const { return buffer_num_frames_; }
size_t num_frames_per_band() const { return num_split_frames_; }

2
webrtc/modules/audio_processing/audio_processing_builder_impl.cc
View File

@ -10,9 +10,9 @@
#include <memory>
#include "api/audio/audio_processing.h"
#include "api/make_ref_counted.h"
#include "modules/audio_processing/audio_processing_impl.h"
#include "modules/audio_processing/include/audio_processing.h"
namespace webrtc {

445
webrtc/modules/audio_processing/audio_processing_impl.cc
View File

@ -14,22 +14,23 @@
#include <cstdint>
#include <cstring>
#include <memory>
#include <optional>
#include <string>
#include <type_traits>
#include <utility>
#include "absl/base/nullability.h"
#include "absl/strings/match.h"
#include "absl/strings/string_view.h"
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/audio/audio_frame.h"
#include "api/task_queue/task_queue_base.h"
#include "common_audio/audio_converter.h"
#include "common_audio/include/audio_util.h"
#include "modules/audio_processing/aec_dump/aec_dump_factory.h"
#include "modules/audio_processing/audio_buffer.h"
#include "modules/audio_processing/include/audio_frame_view.h"
#include "modules/audio_processing/logging/apm_data_dumper.h"
#include "modules/audio_processing/optionally_built_submodule_creators.h"
#include "rtc_base/checks.h"
#include "rtc_base/experiments/field_trial_parser.h"
#include "rtc_base/logging.h"
@ -322,231 +323,6 @@ constexpr int kUnspecifiedDataDumpInputVolume = -100;
// Throughout webrtc, it's assumed that success is represented by zero.
static_assert(AudioProcessing::kNoError == 0, "kNoError must be zero");
absl::optional<AudioProcessingImpl::GainController2ExperimentParams>
AudioProcessingImpl::GetGainController2ExperimentParams() {
constexpr char kFieldTrialName[] = "WebRTC-Audio-GainController2";
if (!field_trial::IsEnabled(kFieldTrialName)) {
return absl::nullopt;
}
FieldTrialFlag enabled("Enabled", false);
// Whether the gain control should switch to AGC2. Enabled by default.
FieldTrialParameter<bool> switch_to_agc2("switch_to_agc2", true);
// AGC2 input volume controller configuration.
constexpr InputVolumeController::Config kDefaultInputVolumeControllerConfig;
FieldTrialConstrained<int> min_input_volume(
"min_input_volume", kDefaultInputVolumeControllerConfig.min_input_volume,
0, 255);
FieldTrialConstrained<int> clipped_level_min(
"clipped_level_min",
kDefaultInputVolumeControllerConfig.clipped_level_min, 0, 255);
FieldTrialConstrained<int> clipped_level_step(
"clipped_level_step",
kDefaultInputVolumeControllerConfig.clipped_level_step, 0, 255);
FieldTrialConstrained<double> clipped_ratio_threshold(
"clipped_ratio_threshold",
kDefaultInputVolumeControllerConfig.clipped_ratio_threshold, 0, 1);
FieldTrialConstrained<int> clipped_wait_frames(
"clipped_wait_frames",
kDefaultInputVolumeControllerConfig.clipped_wait_frames, 0,
absl::nullopt);
FieldTrialParameter<bool> enable_clipping_predictor(
"enable_clipping_predictor",
kDefaultInputVolumeControllerConfig.enable_clipping_predictor);
FieldTrialConstrained<int> target_range_max_dbfs(
"target_range_max_dbfs",
kDefaultInputVolumeControllerConfig.target_range_max_dbfs, -90, 30);
FieldTrialConstrained<int> target_range_min_dbfs(
"target_range_min_dbfs",
kDefaultInputVolumeControllerConfig.target_range_min_dbfs, -90, 30);
FieldTrialConstrained<int> update_input_volume_wait_frames(
"update_input_volume_wait_frames",
kDefaultInputVolumeControllerConfig.update_input_volume_wait_frames, 0,
absl::nullopt);
FieldTrialConstrained<double> speech_probability_threshold(
"speech_probability_threshold",
kDefaultInputVolumeControllerConfig.speech_probability_threshold, 0, 1);
FieldTrialConstrained<double> speech_ratio_threshold(
"speech_ratio_threshold",
kDefaultInputVolumeControllerConfig.speech_ratio_threshold, 0, 1);
// AGC2 adaptive digital controller configuration.
constexpr AudioProcessing::Config::GainController2::AdaptiveDigital
kDefaultAdaptiveDigitalConfig;
FieldTrialConstrained<double> headroom_db(
"headroom_db", kDefaultAdaptiveDigitalConfig.headroom_db, 0,
absl::nullopt);
FieldTrialConstrained<double> max_gain_db(
"max_gain_db", kDefaultAdaptiveDigitalConfig.max_gain_db, 0,
absl::nullopt);
FieldTrialConstrained<double> initial_gain_db(
"initial_gain_db", kDefaultAdaptiveDigitalConfig.initial_gain_db, 0,
absl::nullopt);
FieldTrialConstrained<double> max_gain_change_db_per_second(
"max_gain_change_db_per_second",
kDefaultAdaptiveDigitalConfig.max_gain_change_db_per_second, 0,
absl::nullopt);
FieldTrialConstrained<double> max_output_noise_level_dbfs(
"max_output_noise_level_dbfs",
kDefaultAdaptiveDigitalConfig.max_output_noise_level_dbfs, absl::nullopt,
0);
// Transient suppressor.
FieldTrialParameter<bool> disallow_transient_suppressor_usage(
"disallow_transient_suppressor_usage", false);
// Field-trial based override for the input volume controller and adaptive
// digital configs.
ParseFieldTrial(
{&enabled, &switch_to_agc2, &min_input_volume, &clipped_level_min,
&clipped_level_step, &clipped_ratio_threshold, &clipped_wait_frames,
&enable_clipping_predictor, &target_range_max_dbfs,
&target_range_min_dbfs, &update_input_volume_wait_frames,
&speech_probability_threshold, &speech_ratio_threshold, &headroom_db,
&max_gain_db, &initial_gain_db, &max_gain_change_db_per_second,
&max_output_noise_level_dbfs, &disallow_transient_suppressor_usage},
field_trial::FindFullName(kFieldTrialName));
// Checked already by `IsEnabled()` before parsing, therefore always true.
RTC_DCHECK(enabled);
const bool do_not_change_agc_config = !switch_to_agc2.Get();
if (do_not_change_agc_config && !disallow_transient_suppressor_usage.Get()) {
// Return an unspecifed value since, in this case, both the AGC2 and TS
// configurations won't be adjusted.
return absl::nullopt;
}
using Params = AudioProcessingImpl::GainController2ExperimentParams;
if (do_not_change_agc_config) {
// Return a value that leaves the AGC2 config unchanged and that always
// disables TS.
return Params{.agc2_config = absl::nullopt,
.disallow_transient_suppressor_usage = true};
}
// Return a value that switches all the gain control to AGC2.
return Params{
.agc2_config =
Params::Agc2Config{
.input_volume_controller =
{
.min_input_volume = min_input_volume.Get(),
.clipped_level_min = clipped_level_min.Get(),
.clipped_level_step = clipped_level_step.Get(),
.clipped_ratio_threshold =
static_cast<float>(clipped_ratio_threshold.Get()),
.clipped_wait_frames = clipped_wait_frames.Get(),
.enable_clipping_predictor =
enable_clipping_predictor.Get(),
.target_range_max_dbfs = target_range_max_dbfs.Get(),
.target_range_min_dbfs = target_range_min_dbfs.Get(),
.update_input_volume_wait_frames =
update_input_volume_wait_frames.Get(),
.speech_probability_threshold = static_cast<float>(
speech_probability_threshold.Get()),
.speech_ratio_threshold =
static_cast<float>(speech_ratio_threshold.Get()),
},
.adaptive_digital_controller =
{
.headroom_db = static_cast<float>(headroom_db.Get()),
.max_gain_db = static_cast<float>(max_gain_db.Get()),
.initial_gain_db =
static_cast<float>(initial_gain_db.Get()),
.max_gain_change_db_per_second = static_cast<float>(
max_gain_change_db_per_second.Get()),
.max_output_noise_level_dbfs =
static_cast<float>(max_output_noise_level_dbfs.Get()),
}},
.disallow_transient_suppressor_usage =
disallow_transient_suppressor_usage.Get()};
}
AudioProcessing::Config AudioProcessingImpl::AdjustConfig(
const AudioProcessing::Config& config,
const absl::optional<AudioProcessingImpl::GainController2ExperimentParams>&
experiment_params) {
if (!experiment_params.has_value() ||
(!experiment_params->agc2_config.has_value() &&
!experiment_params->disallow_transient_suppressor_usage)) {
// When the experiment parameters are unspecified or when the AGC and TS
// configuration are not overridden, return the unmodified configuration.
return config;
}
AudioProcessing::Config adjusted_config = config;
// Override the transient suppressor configuration.
if (experiment_params->disallow_transient_suppressor_usage) {
adjusted_config.transient_suppression.enabled = false;
}
// Override the auto gain control configuration if the AGC1 analog gain
// controller is active and `experiment_params->agc2_config` is specified.
const bool agc1_analog_enabled =
config.gain_controller1.enabled &&
(config.gain_controller1.mode ==
AudioProcessing::Config::GainController1::kAdaptiveAnalog ||
config.gain_controller1.analog_gain_controller.enabled);
if (agc1_analog_enabled && experiment_params->agc2_config.has_value()) {
// Check that the unadjusted AGC config meets the preconditions.
const bool hybrid_agc_config_detected =
config.gain_controller1.enabled &&
config.gain_controller1.analog_gain_controller.enabled &&
!config.gain_controller1.analog_gain_controller
.enable_digital_adaptive &&
config.gain_controller2.enabled &&
config.gain_controller2.adaptive_digital.enabled;
const bool full_agc1_config_detected =
config.gain_controller1.enabled &&
config.gain_controller1.analog_gain_controller.enabled &&
config.gain_controller1.analog_gain_controller
.enable_digital_adaptive &&
!config.gain_controller2.enabled;
const bool one_and_only_one_input_volume_controller =
hybrid_agc_config_detected != full_agc1_config_detected;
const bool agc2_input_volume_controller_enabled =
config.gain_controller2.enabled &&
config.gain_controller2.input_volume_controller.enabled;
if (!one_and_only_one_input_volume_controller ||
agc2_input_volume_controller_enabled) {
RTC_LOG(LS_ERROR) << "Cannot adjust AGC config (precondition failed)";
if (!one_and_only_one_input_volume_controller)
RTC_LOG(LS_ERROR)
<< "One and only one input volume controller must be enabled.";
if (agc2_input_volume_controller_enabled)
RTC_LOG(LS_ERROR)
<< "The AGC2 input volume controller must be disabled.";
} else {
adjusted_config.gain_controller1.enabled = false;
adjusted_config.gain_controller1.analog_gain_controller.enabled = false;
adjusted_config.gain_controller2.enabled = true;
adjusted_config.gain_controller2.input_volume_controller.enabled = true;
adjusted_config.gain_controller2.adaptive_digital =
experiment_params->agc2_config->adaptive_digital_controller;
adjusted_config.gain_controller2.adaptive_digital.enabled = true;
}
}
return adjusted_config;
}
bool AudioProcessingImpl::UseApmVadSubModule(
const AudioProcessing::Config& config,
const absl::optional<GainController2ExperimentParams>& experiment_params) {
// The VAD as an APM sub-module is needed only in one case, that is when TS
// and AGC2 are both enabled and when the AGC2 experiment is running and its
// parameters require to fully switch the gain control to AGC2.
return config.transient_suppression.enabled &&
config.gain_controller2.enabled &&
(config.gain_controller2.input_volume_controller.enabled ||
config.gain_controller2.adaptive_digital.enabled) &&
experiment_params.has_value() &&
experiment_params->agc2_config.has_value();
}
AudioProcessingImpl::SubmoduleStates::SubmoduleStates(
bool capture_post_processor_enabled,
bool render_pre_processor_enabled,
@ -561,10 +337,8 @@ bool AudioProcessingImpl::SubmoduleStates::Update(
bool noise_suppressor_enabled,
bool adaptive_gain_controller_enabled,
bool gain_controller2_enabled,
bool voice_activity_detector_enabled,
bool gain_adjustment_enabled,
bool echo_controller_enabled,
bool transient_suppressor_enabled) {
bool echo_controller_enabled) {
bool changed = false;
changed |= (high_pass_filter_enabled != high_pass_filter_enabled_);
changed |=
@ -573,21 +347,16 @@ bool AudioProcessingImpl::SubmoduleStates::Update(
changed |=
(adaptive_gain_controller_enabled != adaptive_gain_controller_enabled_);
changed |= (gain_controller2_enabled != gain_controller2_enabled_);
changed |=
(voice_activity_detector_enabled != voice_activity_detector_enabled_);
changed |= (gain_adjustment_enabled != gain_adjustment_enabled_);
changed |= (echo_controller_enabled != echo_controller_enabled_);
changed |= (transient_suppressor_enabled != transient_suppressor_enabled_);
if (changed) {
high_pass_filter_enabled_ = high_pass_filter_enabled;
mobile_echo_controller_enabled_ = mobile_echo_controller_enabled;
noise_suppressor_enabled_ = noise_suppressor_enabled;
adaptive_gain_controller_enabled_ = adaptive_gain_controller_enabled;
gain_controller2_enabled_ = gain_controller2_enabled;
voice_activity_detector_enabled_ = voice_activity_detector_enabled;
gain_adjustment_enabled_ = gain_adjustment_enabled;
echo_controller_enabled_ = echo_controller_enabled;
transient_suppressor_enabled_ = transient_suppressor_enabled;
}
changed |= first_update_;
@ -664,14 +433,12 @@ AudioProcessingImpl::AudioProcessingImpl(
: data_dumper_(new ApmDataDumper(instance_count_.fetch_add(1) + 1)),
use_setup_specific_default_aec3_config_(
UseSetupSpecificDefaultAec3Congfig()),
gain_controller2_experiment_params_(GetGainController2ExperimentParams()),
transient_suppressor_vad_mode_(TransientSuppressor::VadMode::kDefault),
capture_runtime_settings_(RuntimeSettingQueueSize()),
render_runtime_settings_(RuntimeSettingQueueSize()),
capture_runtime_settings_enqueuer_(&capture_runtime_settings_),
render_runtime_settings_enqueuer_(&render_runtime_settings_),
echo_control_factory_(std::move(echo_control_factory)),
config_(AdjustConfig(config, gain_controller2_experiment_params_)),
config_(config),
submodule_states_(!!capture_post_processor,
!!render_pre_processor,
!!capture_analyzer),
@ -684,8 +451,7 @@ AudioProcessingImpl::AudioProcessingImpl(
!field_trial::IsEnabled(
"WebRTC-ApmExperimentalMultiChannelCaptureKillSwitch"),
EnforceSplitBandHpf(),
MinimizeProcessingForUnusedOutput(),
field_trial::IsEnabled("WebRTC-TransientSuppressorForcedOff")),
MinimizeProcessingForUnusedOutput()),
capture_(),
capture_nonlocked_(),
applied_input_volume_stats_reporter_(
@ -806,12 +572,10 @@ void AudioProcessingImpl::InitializeLocked() {
AllocateRenderQueue();
InitializeGainController1();
InitializeTransientSuppressor();
InitializeHighPassFilter(true);
InitializeResidualEchoDetector();
InitializeEchoController();
InitializeGainController2();
InitializeVoiceActivityDetector();
InitializeNoiseSuppressor();
InitializeAnalyzer();
InitializePostProcessor();
@ -906,52 +670,41 @@ void AudioProcessingImpl::ApplyConfig(const AudioProcessing::Config& config) {
MutexLock lock_render(&mutex_render_);
MutexLock lock_capture(&mutex_capture_);
const auto adjusted_config =
AdjustConfig(config, gain_controller2_experiment_params_);
RTC_LOG(LS_INFO) << "AudioProcessing::ApplyConfig: "
<< adjusted_config.ToString();
RTC_LOG(LS_INFO) << "AudioProcessing::ApplyConfig: " << config.ToString();
const bool pipeline_config_changed =
config_.pipeline.multi_channel_render !=
adjusted_config.pipeline.multi_channel_render ||
config.pipeline.multi_channel_render ||
config_.pipeline.multi_channel_capture !=
adjusted_config.pipeline.multi_channel_capture ||
config.pipeline.multi_channel_capture ||
config_.pipeline.maximum_internal_processing_rate !=
adjusted_config.pipeline.maximum_internal_processing_rate ||
config.pipeline.maximum_internal_processing_rate ||
config_.pipeline.capture_downmix_method !=
adjusted_config.pipeline.capture_downmix_method;
config.pipeline.capture_downmix_method;
const bool aec_config_changed =
config_.echo_canceller.enabled !=
adjusted_config.echo_canceller.enabled ||
config_.echo_canceller.mobile_mode !=
adjusted_config.echo_canceller.mobile_mode;
config_.echo_canceller.enabled != config.echo_canceller.enabled ||
config_.echo_canceller.mobile_mode != config.echo_canceller.mobile_mode;
const bool agc1_config_changed =
config_.gain_controller1 != adjusted_config.gain_controller1;
config_.gain_controller1 != config.gain_controller1;
const bool agc2_config_changed =
config_.gain_controller2 != adjusted_config.gain_controller2;
config_.gain_controller2 != config.gain_controller2;
const bool ns_config_changed =
config_.noise_suppression.enabled !=
adjusted_config.noise_suppression.enabled ||
config_.noise_suppression.level !=
adjusted_config.noise_suppression.level;
const bool ts_config_changed = config_.transient_suppression.enabled !=
adjusted_config.transient_suppression.enabled;
config_.noise_suppression.enabled != config.noise_suppression.enabled ||
config_.noise_suppression.level != config.noise_suppression.level;
const bool pre_amplifier_config_changed =
config_.pre_amplifier.enabled != adjusted_config.pre_amplifier.enabled ||
config_.pre_amplifier.enabled != config.pre_amplifier.enabled ||
config_.pre_amplifier.fixed_gain_factor !=
adjusted_config.pre_amplifier.fixed_gain_factor;
config.pre_amplifier.fixed_gain_factor;
const bool gain_adjustment_config_changed =
config_.capture_level_adjustment !=
adjusted_config.capture_level_adjustment;
config_.capture_level_adjustment != config.capture_level_adjustment;
config_ = adjusted_config;
config_ = config;
if (aec_config_changed) {
InitializeEchoController();
@ -961,10 +714,6 @@ void AudioProcessingImpl::ApplyConfig(const AudioProcessing::Config& config) {
InitializeNoiseSuppressor();
}
if (ts_config_changed) {
InitializeTransientSuppressor();
}
InitializeHighPassFilter(false);
if (agc1_config_changed) {
@ -978,11 +727,8 @@ void AudioProcessingImpl::ApplyConfig(const AudioProcessing::Config& config) {
config_.gain_controller2 = AudioProcessing::Config::GainController2();
}
if (agc2_config_changed || ts_config_changed) {
// AGC2 also depends on TS because of the possible dependency on the APM VAD
// sub-module.
if (agc2_config_changed) {
InitializeGainController2();
InitializeVoiceActivityDetector();
}
if (pre_amplifier_config_changed || gain_adjustment_config_changed) {
@ -996,12 +742,6 @@ void AudioProcessingImpl::ApplyConfig(const AudioProcessing::Config& config) {
}
}
void AudioProcessingImpl::OverrideSubmoduleCreationForTesting(
const ApmSubmoduleCreationOverrides& overrides) {
MutexLock lock(&mutex_capture_);
submodule_creation_overrides_ = overrides;
}
int AudioProcessingImpl::proc_sample_rate_hz() const {
// Used as callback from submodules, hence locking is not allowed.
return capture_nonlocked_.capture_processing_format.sample_rate_hz();
@ -1660,7 +1400,7 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() {
if (submodules_.agc_manager) {
submodules_.agc_manager->Process(*capture_buffer);
absl::optional<int> new_digital_gain =
std::optional<int> new_digital_gain =
submodules_.agc_manager->GetDigitalComressionGain();
if (new_digital_gain && submodules_.gain_control) {
submodules_.gain_control->set_compression_gain_db(*new_digital_gain);
@ -1697,44 +1437,6 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() {
capture_buffer->num_frames()));
}
absl::optional<float> voice_probability;
if (!!submodules_.voice_activity_detector) {
voice_probability = submodules_.voice_activity_detector->Analyze(
AudioFrameView<const float>(capture_buffer->channels(),
capture_buffer->num_channels(),
capture_buffer->num_frames()));
}
if (submodules_.transient_suppressor) {
float transient_suppressor_voice_probability = 1.0f;
switch (transient_suppressor_vad_mode_) {
case TransientSuppressor::VadMode::kDefault:
if (submodules_.agc_manager) {
transient_suppressor_voice_probability =
submodules_.agc_manager->voice_probability();
}
break;
case TransientSuppressor::VadMode::kRnnVad:
RTC_DCHECK(voice_probability.has_value());
transient_suppressor_voice_probability = *voice_probability;
break;
case TransientSuppressor::VadMode::kNoVad:
// The transient suppressor will ignore `voice_probability`.
break;
}
float delayed_voice_probability =
submodules_.transient_suppressor->Suppress(
capture_buffer->channels()[0], capture_buffer->num_frames(),
capture_buffer->num_channels(),
capture_buffer->split_bands_const(0)[kBand0To8kHz],
capture_buffer->num_frames_per_band(),
/*reference_data=*/nullptr, /*reference_length=*/0,
transient_suppressor_voice_probability, capture_.key_pressed);
if (voice_probability.has_value()) {
*voice_probability = delayed_voice_probability;
}
}
// Experimental APM sub-module that analyzes `capture_buffer`.
if (submodules_.capture_analyzer) {
submodules_.capture_analyzer->Analyze(capture_buffer);
@ -1744,8 +1446,8 @@ int AudioProcessingImpl::ProcessCaptureStreamLocked() {
// TODO(bugs.webrtc.org/7494): Let AGC2 detect applied input volume
// changes.
submodules_.gain_controller2->Process(
voice_probability, capture_.applied_input_volume_changed,
capture_buffer);
/*speech_probability=*/std::nullopt,
capture_.applied_input_volume_changed, capture_buffer);
}
if (submodules_.capture_post_processor) {
@ -2023,7 +1725,7 @@ void AudioProcessingImpl::set_stream_analog_level_locked(int level) {
// Invalidate any previously recommended input volume which will be updated by
// `ProcessStream()`.
capture_.recommended_input_volume = absl::nullopt;
capture_.recommended_input_volume = std::nullopt;
if (submodules_.agc_manager) {
submodules_.agc_manager->set_stream_analog_level(level);
@ -2056,7 +1758,7 @@ void AudioProcessingImpl::UpdateRecommendedInputVolumeLocked() {
if (!capture_.applied_input_volume.has_value()) {
// When `set_stream_analog_level()` is not called, no input level can be
// recommended.
capture_.recommended_input_volume = absl::nullopt;
capture_.recommended_input_volume = std::nullopt;
return;
}
@ -2082,9 +1784,10 @@ void AudioProcessingImpl::UpdateRecommendedInputVolumeLocked() {
capture_.recommended_input_volume = capture_.applied_input_volume;
}
bool AudioProcessingImpl::CreateAndAttachAecDump(absl::string_view file_name,
int64_t max_log_size_bytes,
rtc::TaskQueue* worker_queue) {
bool AudioProcessingImpl::CreateAndAttachAecDump(
absl::string_view file_name,
int64_t max_log_size_bytes,
absl::Nonnull<TaskQueueBase*> worker_queue) {
std::unique_ptr<AecDump> aec_dump =
AecDumpFactory::Create(file_name, max_log_size_bytes, worker_queue);
if (!aec_dump) {
@ -2095,9 +1798,10 @@ bool AudioProcessingImpl::CreateAndAttachAecDump(absl::string_view file_name,
return true;
}
bool AudioProcessingImpl::CreateAndAttachAecDump(FILE* handle,
int64_t max_log_size_bytes,
rtc::TaskQueue* worker_queue) {
bool AudioProcessingImpl::CreateAndAttachAecDump(
FILE* handle,
int64_t max_log_size_bytes,
absl::Nonnull<TaskQueueBase*> worker_queue) {
std::unique_ptr<AecDump> aec_dump =
AecDumpFactory::Create(handle, max_log_size_bytes, worker_queue);
if (!aec_dump) {
@ -2142,43 +1846,9 @@ bool AudioProcessingImpl::UpdateActiveSubmoduleStates() {
return submodule_states_.Update(
config_.high_pass_filter.enabled, !!submodules_.echo_control_mobile,
!!submodules_.noise_suppressor, !!submodules_.gain_control,
!!submodules_.gain_controller2, !!submodules_.voice_activity_detector,
!!submodules_.gain_controller2,
config_.pre_amplifier.enabled || config_.capture_level_adjustment.enabled,
capture_nonlocked_.echo_controller_enabled,
!!submodules_.transient_suppressor);
}
void AudioProcessingImpl::InitializeTransientSuppressor() {
// Choose the VAD mode for TS and detect a VAD mode change.
const TransientSuppressor::VadMode previous_vad_mode =
transient_suppressor_vad_mode_;
transient_suppressor_vad_mode_ = TransientSuppressor::VadMode::kDefault;
if (UseApmVadSubModule(config_, gain_controller2_experiment_params_)) {
transient_suppressor_vad_mode_ = TransientSuppressor::VadMode::kRnnVad;
}
const bool vad_mode_changed =
previous_vad_mode != transient_suppressor_vad_mode_;
if (config_.transient_suppression.enabled &&
!constants_.transient_suppressor_forced_off) {
// Attempt to create a transient suppressor, if one is not already created.
if (!submodules_.transient_suppressor || vad_mode_changed) {
submodules_.transient_suppressor = CreateTransientSuppressor(
submodule_creation_overrides_, transient_suppressor_vad_mode_,
proc_fullband_sample_rate_hz(), capture_nonlocked_.split_rate,
num_proc_channels());
if (!submodules_.transient_suppressor) {
RTC_LOG(LS_WARNING)
<< "No transient suppressor created (probably disabled)";
}
} else {
submodules_.transient_suppressor->Initialize(
proc_fullband_sample_rate_hz(), capture_nonlocked_.split_rate,
num_proc_channels());
}
} else {
submodules_.transient_suppressor.reset();
}
capture_nonlocked_.echo_controller_enabled);
}
void AudioProcessingImpl::InitializeHighPassFilter(bool forced_reset) {
@ -2220,7 +1890,7 @@ void AudioProcessingImpl::InitializeEchoController() {
RTC_DCHECK(submodules_.echo_controller);
} else {
EchoCanceller3Config config;
absl::optional<EchoCanceller3Config> multichannel_config;
std::optional<EchoCanceller3Config> multichannel_config;
if (use_setup_specific_default_aec3_config_) {
multichannel_config = EchoCanceller3::CreateDefaultMultichannelConfig();
}
@ -2361,46 +2031,19 @@ void AudioProcessingImpl::InitializeGainController2() {
submodules_.gain_controller2.reset();
return;
}
// Override the input volume controller configuration if the AGC2 experiment
// is running and its parameters require to fully switch the gain control to
// Input volume controller configuration if the AGC2 is running
// and its parameters require to fully switch the gain control to
// AGC2.
const bool input_volume_controller_config_overridden =
gain_controller2_experiment_params_.has_value() &&
gain_controller2_experiment_params_->agc2_config.has_value();
const InputVolumeController::Config input_volume_controller_config =
input_volume_controller_config_overridden
? gain_controller2_experiment_params_->agc2_config
->input_volume_controller
: InputVolumeController::Config{};
// If the APM VAD sub-module is not used, let AGC2 use its internal VAD.
const bool use_internal_vad =
!UseApmVadSubModule(config_, gain_controller2_experiment_params_);
InputVolumeController::Config{};
submodules_.gain_controller2 = std::make_unique<GainController2>(
config_.gain_controller2, input_volume_controller_config,
proc_fullband_sample_rate_hz(), num_proc_channels(), use_internal_vad);
proc_fullband_sample_rate_hz(), num_output_channels(),
/*use_internal_vad=*/true);
submodules_.gain_controller2->SetCaptureOutputUsed(
capture_.capture_output_used);
}
void AudioProcessingImpl::InitializeVoiceActivityDetector() {
if (!UseApmVadSubModule(config_, gain_controller2_experiment_params_)) {
submodules_.voice_activity_detector.reset();
return;
}
if (!submodules_.voice_activity_detector) {
RTC_DCHECK(!!submodules_.gain_controller2);
// TODO(bugs.webrtc.org/13663): Cache CPU features in APM and use here.
submodules_.voice_activity_detector =
std::make_unique<VoiceActivityDetectorWrapper>(
submodules_.gain_controller2->GetCpuFeatures(),
proc_fullband_sample_rate_hz());
} else {
submodules_.voice_activity_detector->Initialize(
proc_fullband_sample_rate_hz());
}
}
void AudioProcessingImpl::InitializeNoiseSuppressor() {
submodules_.noise_suppressor.reset();
@ -2535,8 +2178,6 @@ void AudioProcessingImpl::WriteAecDumpConfigMessage(bool forced) {
apm_config.ns_enabled = config_.noise_suppression.enabled;
apm_config.ns_level = static_cast<int>(config_.noise_suppression.level);
apm_config.transient_suppression_enabled =
config_.transient_suppression.enabled;
apm_config.experiments_description = experiments_description;
apm_config.pre_amplifier_enabled = config_.pre_amplifier.enabled;
apm_config.pre_amplifier_fixed_gain_factor =

112
webrtc/modules/audio_processing/audio_processing_impl.h
View File

@ -16,13 +16,17 @@
#include <atomic>
#include <list>
#include <memory>
#include <optional>
#include <string>
#include <vector>
#include "absl/base/nullability.h"
#include "absl/strings/string_view.h"
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/audio/audio_processing.h"
#include "api/audio/audio_processing_statistics.h"
#include "api/function_view.h"
#include "api/task_queue/task_queue_base.h"
#include "modules/audio_processing/aec3/echo_canceller3.h"
#include "modules/audio_processing/agc/agc_manager_direct.h"
#include "modules/audio_processing/agc/gain_control.h"
@ -35,15 +39,10 @@
#include "modules/audio_processing/high_pass_filter.h"
#include "modules/audio_processing/include/aec_dump.h"
#include "modules/audio_processing/include/audio_frame_proxies.h"
#include "modules/audio_processing/include/audio_processing.h"
#include "modules/audio_processing/include/audio_processing_statistics.h"
#include "modules/audio_processing/ns/noise_suppressor.h"
#include "modules/audio_processing/optionally_built_submodule_creators.h"
#include "modules/audio_processing/render_queue_item_verifier.h"
#include "modules/audio_processing/rms_level.h"
#include "modules/audio_processing/transient/transient_suppressor.h"
#include "rtc_base/gtest_prod_util.h"
#include "rtc_base/ignore_wundef.h"
#include "rtc_base/swap_queue.h"
#include "rtc_base/synchronization/mutex.h"
#include "rtc_base/thread_annotations.h"
@ -72,12 +71,14 @@ class AudioProcessingImpl : public AudioProcessing {
int Initialize() override;
int Initialize(const ProcessingConfig& processing_config) override;
void ApplyConfig(const AudioProcessing::Config& config) override;
bool CreateAndAttachAecDump(absl::string_view file_name,
int64_t max_log_size_bytes,
rtc::TaskQueue* worker_queue) override;
bool CreateAndAttachAecDump(FILE* handle,
int64_t max_log_size_bytes,
rtc::TaskQueue* worker_queue) override;
bool CreateAndAttachAecDump(
absl::string_view file_name,
int64_t max_log_size_bytes,
absl::Nonnull<TaskQueueBase*> worker_queue) override;
bool CreateAndAttachAecDump(
FILE* handle,
int64_t max_log_size_bytes,
absl::Nonnull<TaskQueueBase*> worker_queue) override;
// TODO(webrtc:5298) Deprecated variant.
void AttachAecDump(std::unique_ptr<AecDump> aec_dump) override;
void DetachAecDump() override;
@ -154,24 +155,12 @@ class AudioProcessingImpl : public AudioProcessing {
FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, DefaultBehavior);
FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, ValidConfigBehavior);
FRIEND_TEST_ALL_PREFIXES(ApmConfiguration, InValidConfigBehavior);
FRIEND_TEST_ALL_PREFIXES(ApmWithSubmodulesExcludedTest,
ToggleTransientSuppressor);
FRIEND_TEST_ALL_PREFIXES(ApmWithSubmodulesExcludedTest,
ReinitializeTransientSuppressor);
FRIEND_TEST_ALL_PREFIXES(ApmWithSubmodulesExcludedTest,
BitexactWithDisabledModules);
FRIEND_TEST_ALL_PREFIXES(
AudioProcessingImplGainController2FieldTrialParametrizedTest,
ConfigAdjustedWhenExperimentEnabled);
void set_stream_analog_level_locked(int level)
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
void UpdateRecommendedInputVolumeLocked()
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
void OverrideSubmoduleCreationForTesting(
const ApmSubmoduleCreationOverrides& overrides);
// Class providing thread-safe message pipe functionality for
// `runtime_settings_`.
class RuntimeSettingEnqueuer {
@ -191,49 +180,6 @@ class AudioProcessingImpl : public AudioProcessing {
static std::atomic<int> instance_count_;
const bool use_setup_specific_default_aec3_config_;
// Parameters for the "GainController2" experiment which determines whether
// the following APM sub-modules are created and, if so, their configurations:
// AGC2 (`gain_controller2`), AGC1 (`gain_control`, `agc_manager`) and TS
// (`transient_suppressor`).
// TODO(bugs.webrtc.org/7494): Remove when the "WebRTC-Audio-GainController2"
// field trial is removed.
struct GainController2ExperimentParams {
struct Agc2Config {
InputVolumeController::Config input_volume_controller;
AudioProcessing::Config::GainController2::AdaptiveDigital
adaptive_digital_controller;
};
// When `agc2_config` is specified, all gain control switches to AGC2 and
// the configuration is overridden.
absl::optional<Agc2Config> agc2_config;
// When true, the transient suppressor submodule is never created regardless
// of the APM configuration.
bool disallow_transient_suppressor_usage;
};
// Specified when the "WebRTC-Audio-GainController2" field trial is specified.
// TODO(bugs.webrtc.org/7494): Remove when the "WebRTC-Audio-GainController2"
// field trial is removed.
const absl::optional<GainController2ExperimentParams>
gain_controller2_experiment_params_;
// Parses the "WebRTC-Audio-GainController2" field trial. If disabled, returns
// an unspecified value.
static absl::optional<GainController2ExperimentParams>
GetGainController2ExperimentParams();
// When `experiment_params` is specified, returns an APM configuration
// modified according to the experiment parameters. Otherwise returns
// `config`.
static AudioProcessing::Config AdjustConfig(
const AudioProcessing::Config& config,
const absl::optional<GainController2ExperimentParams>& experiment_params);
// Returns true if the APM VAD sub-module should be used.
static bool UseApmVadSubModule(
const AudioProcessing::Config& config,
const absl::optional<GainController2ExperimentParams>& experiment_params);
TransientSuppressor::VadMode transient_suppressor_vad_mode_;
SwapQueue<RuntimeSetting> capture_runtime_settings_;
SwapQueue<RuntimeSetting> render_runtime_settings_;
@ -254,10 +200,8 @@ class AudioProcessingImpl : public AudioProcessing {
bool noise_suppressor_enabled,
bool adaptive_gain_controller_enabled,
bool gain_controller2_enabled,
bool voice_activity_detector_enabled,
bool gain_adjustment_enabled,
bool echo_controller_enabled,
bool transient_suppressor_enabled);
bool echo_controller_enabled);
bool CaptureMultiBandSubModulesActive() const;
bool CaptureMultiBandProcessingPresent() const;
bool CaptureMultiBandProcessingActive(bool ec_processing_active) const;
@ -276,11 +220,9 @@ class AudioProcessingImpl : public AudioProcessing {
bool mobile_echo_controller_enabled_ = false;
bool noise_suppressor_enabled_ = false;
bool adaptive_gain_controller_enabled_ = false;
bool voice_activity_detector_enabled_ = false;
bool gain_controller2_enabled_ = false;
bool gain_adjustment_enabled_ = false;
bool echo_controller_enabled_ = false;
bool transient_suppressor_enabled_ = false;
bool first_update_ = true;
};
@ -317,18 +259,9 @@ class AudioProcessingImpl : public AudioProcessing {
void InitializeHighPassFilter(bool forced_reset)
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
void InitializeGainController1() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
void InitializeTransientSuppressor()
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
// Initializes the `GainController2` sub-module. If the sub-module is enabled,
// recreates it.
void InitializeGainController2() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
// Initializes the `VoiceActivityDetectorWrapper` sub-module. If the
// sub-module is enabled, recreates it. Call `InitializeGainController2()`
// first.
// TODO(bugs.webrtc.org/13663): Remove if TS is removed otherwise remove call
// order requirement - i.e., decouple from `InitializeGainController2()`.
void InitializeVoiceActivityDetector()
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
void InitializeNoiseSuppressor() RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
void InitializeCaptureLevelsAdjuster()
RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_capture_);
@ -423,10 +356,6 @@ class AudioProcessingImpl : public AudioProcessing {
// Struct containing the Config specifying the behavior of APM.
AudioProcessing::Config config_;
// Overrides for testing the exclusion of some submodules from the build.
ApmSubmoduleCreationOverrides submodule_creation_overrides_
RTC_GUARDED_BY(mutex_capture_);
// Class containing information about what submodules are active.
SubmoduleStates submodule_states_;
@ -448,12 +377,10 @@ class AudioProcessingImpl : public AudioProcessing {
std::unique_ptr<AgcManagerDirect> agc_manager;
std::unique_ptr<GainControlImpl> gain_control;
std::unique_ptr<GainController2> gain_controller2;
std::unique_ptr<VoiceActivityDetectorWrapper> voice_activity_detector;
std::unique_ptr<HighPassFilter> high_pass_filter;
std::unique_ptr<EchoControl> echo_controller;
std::unique_ptr<EchoControlMobileImpl> echo_control_mobile;
std::unique_ptr<NoiseSuppressor> noise_suppressor;
std::unique_ptr<TransientSuppressor> transient_suppressor;
std::unique_ptr<CaptureLevelsAdjuster> capture_levels_adjuster;
} submodules_;
@ -479,19 +406,16 @@ class AudioProcessingImpl : public AudioProcessing {
ApmConstants(bool multi_channel_render_support,
bool multi_channel_capture_support,
bool enforce_split_band_hpf,
bool minimize_processing_for_unused_output,
bool transient_suppressor_forced_off)
bool minimize_processing_for_unused_output)
: multi_channel_render_support(multi_channel_render_support),
multi_channel_capture_support(multi_channel_capture_support),
enforce_split_band_hpf(enforce_split_band_hpf),
minimize_processing_for_unused_output(
minimize_processing_for_unused_output),
transient_suppressor_forced_off(transient_suppressor_forced_off) {}
minimize_processing_for_unused_output) {}
bool multi_channel_render_support;
bool multi_channel_capture_support;
bool enforce_split_band_hpf;
bool minimize_processing_for_unused_output;
bool transient_suppressor_forced_off;
} constants_;
struct ApmCaptureState {
@ -516,12 +440,12 @@ class AudioProcessingImpl : public AudioProcessing {
AudioProcessingStats stats;
// Input volume applied on the audio input device when the audio is
// acquired. Unspecified when unknown.
absl::optional<int> applied_input_volume;
std::optional<int> applied_input_volume;
bool applied_input_volume_changed;
// Recommended input volume to apply on the audio input device the next time
// that audio is acquired. Unspecified when no input volume can be
// recommended.
absl::optional<int> recommended_input_volume;
std::optional<int> recommended_input_volume;
} capture_ RTC_GUARDED_BY(mutex_capture_);
struct ApmCaptureNonLockedState {

2
webrtc/modules/audio_processing/echo_control_mobile_impl.cc
View File

@ -14,9 +14,9 @@
#include <cstdint>
#include "api/audio/audio_processing.h"
#include "modules/audio_processing/aecm/echo_control_mobile.h"
#include "modules/audio_processing/audio_buffer.h"
#include "modules/audio_processing/include/audio_processing.h"
#include "rtc_base/checks.h"
namespace webrtc {

4
webrtc/modules/audio_processing/echo_detector/circular_buffer.cc
View File

@ -28,9 +28,9 @@ void CircularBuffer::Push(float value) {
RTC_DCHECK_LE(nr_elements_in_buffer_, buffer_.size());
}
absl::optional<float> CircularBuffer::Pop() {
std::optional<float> CircularBuffer::Pop() {
if (nr_elements_in_buffer_ == 0) {
return absl::nullopt;
return std::nullopt;
}
const size_t index =
(buffer_.size() + next_insertion_index_ - nr_elements_in_buffer_) %

5
webrtc/modules/audio_processing/echo_detector/circular_buffer.h
View File

@ -13,10 +13,9 @@
#include <stddef.h>
#include <optional>
#include <vector>
#include "absl/types/optional.h"
namespace webrtc {
// Ring buffer containing floating point values.
@ -26,7 +25,7 @@ struct CircularBuffer {
~CircularBuffer();
void Push(float value);
absl::optional<float> Pop();
std::optional<float> Pop();
size_t Size() const { return nr_elements_in_buffer_; }
// This function fills the buffer with zeros, but does not change its size.
void Clear();

4
webrtc/modules/audio_processing/gain_control_impl.cc
View File

@ -11,11 +11,11 @@
#include "modules/audio_processing/gain_control_impl.h"
#include <cstdint>
#include <optional>
#include "absl/types/optional.h"
#include "api/audio/audio_processing.h"
#include "modules/audio_processing/agc/legacy/gain_control.h"
#include "modules/audio_processing/audio_buffer.h"
#include "modules/audio_processing/include/audio_processing.h"
#include "modules/audio_processing/logging/apm_data_dumper.h"
#include "rtc_base/checks.h"
#include "rtc_base/logging.h"

6
webrtc/modules/audio_processing/gain_control_impl.h
View File

@ -15,9 +15,9 @@
#include <stdint.h>
#include <memory>
#include <optional>
#include <vector>
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "modules/audio_processing/agc/gain_control.h"
@ -81,8 +81,8 @@ class GainControlImpl : public GainControl {
std::vector<std::unique_ptr<MonoAgcState>> mono_agcs_;
std::vector<int> capture_levels_;
absl::optional<size_t> num_proc_channels_;
absl::optional<int> sample_rate_hz_;
std::optional<size_t> num_proc_channels_;
std::optional<int> sample_rate_hz_;
static int instance_counter_;
};

27
webrtc/modules/audio_processing/gain_controller2.cc
View File

@ -13,6 +13,7 @@
#include <memory>
#include <utility>
#include "api/audio/audio_frame.h"
#include "common_audio/include/audio_util.h"
#include "modules/audio_processing/agc2/agc2_common.h"
#include "modules/audio_processing/agc2/cpu_features.h"
@ -63,11 +64,11 @@ struct SpeechLevel {
};
// Computes the audio levels for the first channel in `frame`.
AudioLevels ComputeAudioLevels(AudioFrameView<float> frame,
AudioLevels ComputeAudioLevels(DeinterleavedView<float> frame,
ApmDataDumper& data_dumper) {
float peak = 0.0f;
float rms = 0.0f;
for (const auto& x : frame.channel(0)) {
for (const auto& x : frame[0]) {
peak = std::max(std::fabs(x), peak);
rms += x * x;
}
@ -94,7 +95,9 @@ GainController2::GainController2(
fixed_gain_applier_(
/*hard_clip_samples=*/false,
/*initial_gain_factor=*/DbToRatio(config.fixed_digital.gain_db)),
limiter_(sample_rate_hz, &data_dumper_, /*histogram_name_prefix=*/"Agc2"),
limiter_(&data_dumper_,
SampleRateToDefaultChannelSize(sample_rate_hz),
/*histogram_name_prefix=*/"Agc2"),
calls_since_last_limiter_log_(0) {
RTC_DCHECK(Validate(config));
data_dumper_.InitiateNewSetOfRecordings();
@ -153,7 +156,7 @@ void GainController2::SetFixedGainDb(float gain_db) {
void GainController2::Analyze(int applied_input_volume,
const AudioBuffer& audio_buffer) {
recommended_input_volume_ = absl::nullopt;
recommended_input_volume_ = std::nullopt;
RTC_DCHECK_GE(applied_input_volume, 0);
RTC_DCHECK_LE(applied_input_volume, 255);
@ -164,10 +167,10 @@ void GainController2::Analyze(int applied_input_volume,
}
}
void GainController2::Process(absl::optional<float> speech_probability,
void GainController2::Process(std::optional<float> speech_probability,
bool input_volume_changed,
AudioBuffer* audio) {
recommended_input_volume_ = absl::nullopt;
recommended_input_volume_ = std::nullopt;
data_dumper_.DumpRaw("agc2_applied_input_volume_changed",
input_volume_changed);
@ -179,8 +182,8 @@ void GainController2::Process(absl::optional<float> speech_probability,
saturation_protector_->Reset();
}
AudioFrameView<float> float_frame(audio->channels(), audio->num_channels(),
audio->num_frames());
DeinterleavedView<float> float_frame = audio->view();
// Compute speech probability.
if (vad_) {
// When the VAD component runs, `speech_probability` should not be specified
@ -200,13 +203,13 @@ void GainController2::Process(absl::optional<float> speech_probability,
// Compute audio, noise and speech levels.
AudioLevels audio_levels = ComputeAudioLevels(float_frame, data_dumper_);
absl::optional<float> noise_rms_dbfs;
std::optional<float> noise_rms_dbfs;
if (noise_level_estimator_) {
// TODO(bugs.webrtc.org/7494): Pass `audio_levels` to remove duplicated
// computation in `noise_level_estimator_`.
noise_rms_dbfs = noise_level_estimator_->Analyze(float_frame);
}
absl::optional<SpeechLevel> speech_level;
std::optional<SpeechLevel> speech_level;
if (speech_level_estimator_) {
RTC_DCHECK(speech_probability.has_value());
speech_level_estimator_->Update(
@ -225,8 +228,8 @@ void GainController2::Process(absl::optional<float> speech_probability,
input_volume_controller_->RecommendInputVolume(
*speech_probability,
speech_level->is_confident
? absl::optional<float>(speech_level->rms_dbfs)
: absl::nullopt);
? std::optional<float>(speech_level->rms_dbfs)
: std::nullopt);
}
}

9
webrtc/modules/audio_processing/gain_controller2.h
View File

@ -15,6 +15,7 @@
#include <memory>
#include <string>
#include "api/audio/audio_processing.h"
#include "modules/audio_processing/agc2/adaptive_digital_gain_controller.h"
#include "modules/audio_processing/agc2/cpu_features.h"
#include "modules/audio_processing/agc2/gain_applier.h"
@ -24,7 +25,6 @@
#include "modules/audio_processing/agc2/saturation_protector.h"
#include "modules/audio_processing/agc2/speech_level_estimator.h"
#include "modules/audio_processing/agc2/vad_wrapper.h"
#include "modules/audio_processing/include/audio_processing.h"
#include "modules/audio_processing/logging/apm_data_dumper.h"
namespace webrtc {
@ -68,7 +68,8 @@ class GainController2 {
// computes the speech probability via `vad_`.
// Handles input volume changes; if the caller cannot determine whether an
// input volume change occurred, set `input_volume_changed` to false.
void Process(absl::optional<float> speech_probability,
// TODO(bugs.webrtc.org/7494): Remove `speech_probability`.
void Process(std::optional<float> speech_probability,
bool input_volume_changed,
AudioBuffer* audio);
@ -76,7 +77,7 @@ class GainController2 {
AvailableCpuFeatures GetCpuFeatures() const { return cpu_features_; }
absl::optional<int> recommended_input_volume() const {
std::optional<int> recommended_input_volume() const {
return recommended_input_volume_;
}
@ -102,7 +103,7 @@ class GainController2 {
// Recommended input volume from `InputVolumecontroller`. Non-empty after
// `Process()` if input volume controller is enabled and
// `InputVolumeController::Process()` has returned a non-empty value.
absl::optional<int> recommended_input_volume_;
std::optional<int> recommended_input_volume_;
};
} // namespace webrtc

6
webrtc/modules/audio_processing/include/aec_dump.h
View File

@ -13,12 +13,12 @@
#include <stdint.h>
#include <optional>
#include <string>
#include "absl/base/attributes.h"
#include "absl/types/optional.h"
#include "api/audio/audio_processing.h"
#include "modules/audio_processing/include/audio_frame_view.h"
#include "modules/audio_processing/include/audio_processing.h"
namespace webrtc {
@ -68,7 +68,7 @@ class AecDump {
struct AudioProcessingState {
int delay;
int drift;
absl::optional<int> applied_input_volume;
std::optional<int> applied_input_volume;
bool keypress;
};

2
webrtc/modules/audio_processing/include/audio_frame_proxies.cc
View File

@ -11,7 +11,7 @@
#include "modules/audio_processing/include/audio_frame_proxies.h"
#include "api/audio/audio_frame.h"
#include "modules/audio_processing/include/audio_processing.h"
#include "api/audio/audio_processing.h"
namespace webrtc {

60
webrtc/modules/audio_processing/include/audio_frame_view.h
View File

@ -11,7 +11,7 @@
#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_FRAME_VIEW_H_
#define MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_FRAME_VIEW_H_
#include "api/array_view.h"
#include "api/audio/audio_view.h"
namespace webrtc {
@ -22,46 +22,44 @@ class AudioFrameView {
// `num_channels` and `channel_size` describe the T**
// `audio_samples`. `audio_samples` is assumed to point to a
// two-dimensional |num_channels * channel_size| array of floats.
//
// Note: The implementation now only requires the first channel pointer.
// The previous implementation retained a pointer to externally owned array
// of channel pointers, but since the channel size and count are provided
// and the array is assumed to be a single two-dimensional array, the other
// channel pointers can be calculated based on that (which is what the class
// now uses `DeinterleavedView<>` internally for).
AudioFrameView(T* const* audio_samples, int num_channels, int channel_size)
: audio_samples_(audio_samples),
num_channels_(num_channels),
channel_size_(channel_size) {
RTC_DCHECK_GE(num_channels_, 0);
RTC_DCHECK_GE(channel_size_, 0);
: view_(num_channels && channel_size ? audio_samples[0] : nullptr,
channel_size,
num_channels) {
RTC_DCHECK_GE(view_.num_channels(), 0);
RTC_DCHECK_GE(view_.samples_per_channel(), 0);
}
// Implicit cast to allow converting Frame<float> to
// Frame<const float>.
// Implicit cast to allow converting AudioFrameView<float> to
// AudioFrameView<const float>.
template <class U>
AudioFrameView(AudioFrameView<U> other)
: audio_samples_(other.data()),
num_channels_(other.num_channels()),
channel_size_(other.samples_per_channel()) {}
AudioFrameView(AudioFrameView<U> other) : view_(other.view()) {}
// Allow constructing AudioFrameView from a DeinterleavedView.
template <class U>
explicit AudioFrameView(DeinterleavedView<U> view) : view_(view) {}
AudioFrameView() = delete;
int num_channels() const { return num_channels_; }
int num_channels() const { return view_.num_channels(); }
int samples_per_channel() const { return view_.samples_per_channel(); }
MonoView<T> channel(int idx) { return view_[idx]; }
MonoView<const T> channel(int idx) const { return view_[idx]; }
MonoView<T> operator[](int idx) { return view_[idx]; }
MonoView<const T> operator[](int idx) const { return view_[idx]; }
int samples_per_channel() const { return channel_size_; }
rtc::ArrayView<T> channel(int idx) {
RTC_DCHECK_LE(0, idx);
RTC_DCHECK_LE(idx, num_channels_);
return rtc::ArrayView<T>(audio_samples_[idx], channel_size_);
}
rtc::ArrayView<const T> channel(int idx) const {
RTC_DCHECK_LE(0, idx);
RTC_DCHECK_LE(idx, num_channels_);
return rtc::ArrayView<const T>(audio_samples_[idx], channel_size_);
}
T* const* data() { return audio_samples_; }
DeinterleavedView<T> view() { return view_; }
DeinterleavedView<const T> view() const { return view_; }
private:
T* const* audio_samples_;
int num_channels_;
int channel_size_;
DeinterleavedView<T> view_;
};
} // namespace webrtc

210
webrtc/modules/audio_processing/include/audio_processing.cc
View File

@ -1,210 +0,0 @@
/*
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/audio_processing/include/audio_processing.h"
#include "rtc_base/strings/string_builder.h"
#include "rtc_base/system/arch.h"
namespace webrtc {
namespace {
using Agc1Config = AudioProcessing::Config::GainController1;
using Agc2Config = AudioProcessing::Config::GainController2;
std::string NoiseSuppressionLevelToString(
const AudioProcessing::Config::NoiseSuppression::Level& level) {
switch (level) {
case AudioProcessing::Config::NoiseSuppression::Level::kLow:
return "Low";
case AudioProcessing::Config::NoiseSuppression::Level::kModerate:
return "Moderate";
case AudioProcessing::Config::NoiseSuppression::Level::kHigh:
return "High";
case AudioProcessing::Config::NoiseSuppression::Level::kVeryHigh:
return "VeryHigh";
}
RTC_CHECK_NOTREACHED();
}
std::string GainController1ModeToString(const Agc1Config::Mode& mode) {
switch (mode) {
case Agc1Config::Mode::kAdaptiveAnalog:
return "AdaptiveAnalog";
case Agc1Config::Mode::kAdaptiveDigital:
return "AdaptiveDigital";
case Agc1Config::Mode::kFixedDigital:
return "FixedDigital";
}
RTC_CHECK_NOTREACHED();
}
} // namespace
constexpr int AudioProcessing::kNativeSampleRatesHz[];
void CustomProcessing::SetRuntimeSetting(
AudioProcessing::RuntimeSetting setting) {}
bool Agc1Config::operator==(const Agc1Config& rhs) const {
const auto& analog_lhs = analog_gain_controller;
const auto& analog_rhs = rhs.analog_gain_controller;
return enabled == rhs.enabled && mode == rhs.mode &&
target_level_dbfs == rhs.target_level_dbfs &&
compression_gain_db == rhs.compression_gain_db &&
enable_limiter == rhs.enable_limiter &&
analog_lhs.enabled == analog_rhs.enabled &&
analog_lhs.startup_min_volume == analog_rhs.startup_min_volume &&
analog_lhs.clipped_level_min == analog_rhs.clipped_level_min &&
analog_lhs.enable_digital_adaptive ==
analog_rhs.enable_digital_adaptive &&
analog_lhs.clipped_level_step == analog_rhs.clipped_level_step &&
analog_lhs.clipped_ratio_threshold ==
analog_rhs.clipped_ratio_threshold &&
analog_lhs.clipped_wait_frames == analog_rhs.clipped_wait_frames &&
analog_lhs.clipping_predictor.mode ==
analog_rhs.clipping_predictor.mode &&
analog_lhs.clipping_predictor.window_length ==
analog_rhs.clipping_predictor.window_length &&
analog_lhs.clipping_predictor.reference_window_length ==
analog_rhs.clipping_predictor.reference_window_length &&
analog_lhs.clipping_predictor.reference_window_delay ==
analog_rhs.clipping_predictor.reference_window_delay &&
analog_lhs.clipping_predictor.clipping_threshold ==
analog_rhs.clipping_predictor.clipping_threshold &&
analog_lhs.clipping_predictor.crest_factor_margin ==
analog_rhs.clipping_predictor.crest_factor_margin &&
analog_lhs.clipping_predictor.use_predicted_step ==
analog_rhs.clipping_predictor.use_predicted_step;
}
bool Agc2Config::AdaptiveDigital::operator==(
const Agc2Config::AdaptiveDigital& rhs) const {
return enabled == rhs.enabled && headroom_db == rhs.headroom_db &&
max_gain_db == rhs.max_gain_db &&
initial_gain_db == rhs.initial_gain_db &&
max_gain_change_db_per_second == rhs.max_gain_change_db_per_second &&
max_output_noise_level_dbfs == rhs.max_output_noise_level_dbfs;
}
bool Agc2Config::InputVolumeController::operator==(
const Agc2Config::InputVolumeController& rhs) const {
return enabled == rhs.enabled;
}
bool Agc2Config::operator==(const Agc2Config& rhs) const {
return enabled == rhs.enabled &&
fixed_digital.gain_db == rhs.fixed_digital.gain_db &&
adaptive_digital == rhs.adaptive_digital &&
input_volume_controller == rhs.input_volume_controller;
}
bool AudioProcessing::Config::CaptureLevelAdjustment::operator==(
const AudioProcessing::Config::CaptureLevelAdjustment& rhs) const {
return enabled == rhs.enabled && pre_gain_factor == rhs.pre_gain_factor &&
post_gain_factor == rhs.post_gain_factor &&
analog_mic_gain_emulation == rhs.analog_mic_gain_emulation;
}
bool AudioProcessing::Config::CaptureLevelAdjustment::AnalogMicGainEmulation::
operator==(const AudioProcessing::Config::CaptureLevelAdjustment::
AnalogMicGainEmulation& rhs) const {
return enabled == rhs.enabled && initial_level == rhs.initial_level;
}
std::string AudioProcessing::Config::ToString() const {
char buf[2048];
rtc::SimpleStringBuilder builder(buf);
builder << "AudioProcessing::Config{ "
"pipeline: { "
"maximum_internal_processing_rate: "
<< pipeline.maximum_internal_processing_rate
<< ", multi_channel_render: " << pipeline.multi_channel_render
<< ", multi_channel_capture: " << pipeline.multi_channel_capture
<< " }, pre_amplifier: { enabled: " << pre_amplifier.enabled
<< ", fixed_gain_factor: " << pre_amplifier.fixed_gain_factor
<< " },capture_level_adjustment: { enabled: "
<< capture_level_adjustment.enabled
<< ", pre_gain_factor: " << capture_level_adjustment.pre_gain_factor
<< ", post_gain_factor: " << capture_level_adjustment.post_gain_factor
<< ", analog_mic_gain_emulation: { enabled: "
<< capture_level_adjustment.analog_mic_gain_emulation.enabled
<< ", initial_level: "
<< capture_level_adjustment.analog_mic_gain_emulation.initial_level
<< " }}, high_pass_filter: { enabled: " << high_pass_filter.enabled
<< " }, echo_canceller: { enabled: " << echo_canceller.enabled
<< ", mobile_mode: " << echo_canceller.mobile_mode
<< ", enforce_high_pass_filtering: "
<< echo_canceller.enforce_high_pass_filtering
<< " }, noise_suppression: { enabled: " << noise_suppression.enabled
<< ", level: "
<< NoiseSuppressionLevelToString(noise_suppression.level)
<< " }, transient_suppression: { enabled: "
<< transient_suppression.enabled
<< " }, gain_controller1: { enabled: " << gain_controller1.enabled
<< ", mode: " << GainController1ModeToString(gain_controller1.mode)
<< ", target_level_dbfs: " << gain_controller1.target_level_dbfs
<< ", compression_gain_db: " << gain_controller1.compression_gain_db
<< ", enable_limiter: " << gain_controller1.enable_limiter
<< ", analog_gain_controller { enabled: "
<< gain_controller1.analog_gain_controller.enabled
<< ", startup_min_volume: "
<< gain_controller1.analog_gain_controller.startup_min_volume
<< ", clipped_level_min: "
<< gain_controller1.analog_gain_controller.clipped_level_min
<< ", enable_digital_adaptive: "
<< gain_controller1.analog_gain_controller.enable_digital_adaptive
<< ", clipped_level_step: "
<< gain_controller1.analog_gain_controller.clipped_level_step
<< ", clipped_ratio_threshold: "
<< gain_controller1.analog_gain_controller.clipped_ratio_threshold
<< ", clipped_wait_frames: "
<< gain_controller1.analog_gain_controller.clipped_wait_frames
<< ", clipping_predictor: { enabled: "
<< gain_controller1.analog_gain_controller.clipping_predictor.enabled
<< ", mode: "
<< gain_controller1.analog_gain_controller.clipping_predictor.mode
<< ", window_length: "
<< gain_controller1.analog_gain_controller.clipping_predictor
.window_length
<< ", reference_window_length: "
<< gain_controller1.analog_gain_controller.clipping_predictor
.reference_window_length
<< ", reference_window_delay: "
<< gain_controller1.analog_gain_controller.clipping_predictor
.reference_window_delay
<< ", clipping_threshold: "
<< gain_controller1.analog_gain_controller.clipping_predictor
.clipping_threshold
<< ", crest_factor_margin: "
<< gain_controller1.analog_gain_controller.clipping_predictor
.crest_factor_margin
<< ", use_predicted_step: "
<< gain_controller1.analog_gain_controller.clipping_predictor
.use_predicted_step
<< " }}}, gain_controller2: { enabled: " << gain_controller2.enabled
<< ", fixed_digital: { gain_db: "
<< gain_controller2.fixed_digital.gain_db
<< " }, adaptive_digital: { enabled: "
<< gain_controller2.adaptive_digital.enabled
<< ", headroom_db: " << gain_controller2.adaptive_digital.headroom_db
<< ", max_gain_db: " << gain_controller2.adaptive_digital.max_gain_db
<< ", initial_gain_db: "
<< gain_controller2.adaptive_digital.initial_gain_db
<< ", max_gain_change_db_per_second: "
<< gain_controller2.adaptive_digital.max_gain_change_db_per_second
<< ", max_output_noise_level_dbfs: "
<< gain_controller2.adaptive_digital.max_output_noise_level_dbfs
<< " }, input_volume_control : { enabled "
<< gain_controller2.input_volume_controller.enabled << "}}";
return builder.str();
}
} // namespace webrtc

929
webrtc/modules/audio_processing/include/audio_processing.h
View File

@ -11,931 +11,8 @@
#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_
#define MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_
// MSVC++ requires this to be set before any other includes to get M_PI.
#ifndef _USE_MATH_DEFINES
#define _USE_MATH_DEFINES
#endif
#include <math.h>
#include <stddef.h> // size_t
#include <stdio.h> // FILE
#include <string.h>
#include <vector>
#include "absl/strings/string_view.h"
#include "absl/types/optional.h"
#include "api/array_view.h"
#include "api/audio/echo_canceller3_config.h"
#include "api/audio/echo_control.h"
#include "api/scoped_refptr.h"
#include "modules/audio_processing/include/audio_processing_statistics.h"
#include "rtc_base/arraysize.h"
#include "rtc_base/ref_count.h"
#include "rtc_base/system/file_wrapper.h"
#include "rtc_base/system/rtc_export.h"
namespace rtc {
class TaskQueue;
} // namespace rtc
namespace webrtc {
class AecDump;
class AudioBuffer;
class StreamConfig;
class ProcessingConfig;
class EchoDetector;
class CustomAudioAnalyzer;
class CustomProcessing;
// The Audio Processing Module (APM) provides a collection of voice processing
// components designed for real-time communications software.
//
// APM operates on two audio streams on a frame-by-frame basis. Frames of the
// primary stream, on which all processing is applied, are passed to
// `ProcessStream()`. Frames of the reverse direction stream are passed to
// `ProcessReverseStream()`. On the client-side, this will typically be the
// near-end (capture) and far-end (render) streams, respectively. APM should be
// placed in the signal chain as close to the audio hardware abstraction layer
// (HAL) as possible.
//
// On the server-side, the reverse stream will normally not be used, with
// processing occurring on each incoming stream.
//
// Component interfaces follow a similar pattern and are accessed through
// corresponding getters in APM. All components are disabled at create-time,
// with default settings that are recommended for most situations. New settings
// can be applied without enabling a component. Enabling a component triggers
// memory allocation and initialization to allow it to start processing the
// streams.
//
// Thread safety is provided with the following assumptions to reduce locking
// overhead:
// 1. The stream getters and setters are called from the same thread as
// ProcessStream(). More precisely, stream functions are never called
// concurrently with ProcessStream().
// 2. Parameter getters are never called concurrently with the corresponding
// setter.
//
// APM accepts only linear PCM audio data in chunks of ~10 ms (see
// AudioProcessing::GetFrameSize() for details) and sample rates ranging from
// 8000 Hz to 384000 Hz. The int16 interfaces use interleaved data, while the
// float interfaces use deinterleaved data.
//
// Usage example, omitting error checking:
// rtc::scoped_refptr<AudioProcessing> apm = AudioProcessingBuilder().Create();
//
// AudioProcessing::Config config;
// config.echo_canceller.enabled = true;
// config.echo_canceller.mobile_mode = false;
//
// config.gain_controller1.enabled = true;
// config.gain_controller1.mode =
// AudioProcessing::Config::GainController1::kAdaptiveAnalog;
// config.gain_controller1.analog_level_minimum = 0;
// config.gain_controller1.analog_level_maximum = 255;
//
// config.gain_controller2.enabled = true;
//
// config.high_pass_filter.enabled = true;
//
// apm->ApplyConfig(config)
//
// // Start a voice call...
//
// // ... Render frame arrives bound for the audio HAL ...
// apm->ProcessReverseStream(render_frame);
//
// // ... Capture frame arrives from the audio HAL ...
// // Call required set_stream_ functions.
// apm->set_stream_delay_ms(delay_ms);
// apm->set_stream_analog_level(analog_level);
//
// apm->ProcessStream(capture_frame);
//
// // Call required stream_ functions.
// analog_level = apm->recommended_stream_analog_level();
// has_voice = apm->stream_has_voice();
//
// // Repeat render and capture processing for the duration of the call...
// // Start a new call...
// apm->Initialize();
//
// // Close the application...
// apm.reset();
//
class RTC_EXPORT AudioProcessing : public rtc::RefCountInterface {
public:
// The struct below constitutes the new parameter scheme for the audio
// processing. It is being introduced gradually and until it is fully
// introduced, it is prone to change.
// TODO(peah): Remove this comment once the new config scheme is fully rolled
// out.
//
// The parameters and behavior of the audio processing module are controlled
// by changing the default values in the AudioProcessing::Config struct.
// The config is applied by passing the struct to the ApplyConfig method.
//
// This config is intended to be used during setup, and to enable/disable
// top-level processing effects. Use during processing may cause undesired
// submodule resets, affecting the audio quality. Use the RuntimeSetting
// construct for runtime configuration.
struct RTC_EXPORT Config {
// Sets the properties of the audio processing pipeline.
struct RTC_EXPORT Pipeline {
// Ways to downmix a multi-channel track to mono.
enum class DownmixMethod {
kAverageChannels, // Average across channels.
kUseFirstChannel // Use the first channel.
};
// Maximum allowed processing rate used internally. May only be set to
// 32000 or 48000 and any differing values will be treated as 48000.
int maximum_internal_processing_rate = 48000;
// Allow multi-channel processing of render audio.
bool multi_channel_render = false;
// Allow multi-channel processing of capture audio when AEC3 is active
// or a custom AEC is injected..
bool multi_channel_capture = false;
// Indicates how to downmix multi-channel capture audio to mono (when
// needed).
DownmixMethod capture_downmix_method = DownmixMethod::kAverageChannels;
} pipeline;
// Enabled the pre-amplifier. It amplifies the capture signal
// before any other processing is done.
// TODO(webrtc:5298): Deprecate and use the pre-gain functionality in
// capture_level_adjustment instead.
struct PreAmplifier {
bool enabled = false;
float fixed_gain_factor = 1.0f;
} pre_amplifier;
// Functionality for general level adjustment in the capture pipeline. This
// should not be used together with the legacy PreAmplifier functionality.
struct CaptureLevelAdjustment {
bool operator==(const CaptureLevelAdjustment& rhs) const;
bool operator!=(const CaptureLevelAdjustment& rhs) const {
return !(*this == rhs);
}
bool enabled = false;
// The `pre_gain_factor` scales the signal before any processing is done.
float pre_gain_factor = 1.0f;
// The `post_gain_factor` scales the signal after all processing is done.
float post_gain_factor = 1.0f;
struct AnalogMicGainEmulation {
bool operator==(const AnalogMicGainEmulation& rhs) const;
bool operator!=(const AnalogMicGainEmulation& rhs) const {
return !(*this == rhs);
}
bool enabled = false;
// Initial analog gain level to use for the emulated analog gain. Must
// be in the range [0...255].
int initial_level = 255;
} analog_mic_gain_emulation;
} capture_level_adjustment;
struct HighPassFilter {
bool enabled = false;
bool apply_in_full_band = true;
} high_pass_filter;
struct EchoCanceller {
bool enabled = false;
bool mobile_mode = false;
bool export_linear_aec_output = false;
// Enforce the highpass filter to be on (has no effect for the mobile
// mode).
bool enforce_high_pass_filtering = true;
} echo_canceller;
// Enables background noise suppression.
struct NoiseSuppression {
bool enabled = false;
enum Level { kLow, kModerate, kHigh, kVeryHigh };
Level level = kModerate;
bool analyze_linear_aec_output_when_available = false;
} noise_suppression;
// Enables transient suppression.
struct TransientSuppression {
bool enabled = false;
} transient_suppression;
// Enables automatic gain control (AGC) functionality.
// The automatic gain control (AGC) component brings the signal to an
// appropriate range. This is done by applying a digital gain directly and,
// in the analog mode, prescribing an analog gain to be applied at the audio
// HAL.
// Recommended to be enabled on the client-side.
struct RTC_EXPORT GainController1 {
bool operator==(const GainController1& rhs) const;
bool operator!=(const GainController1& rhs) const {
return !(*this == rhs);
}
bool enabled = false;
enum Mode {
// Adaptive mode intended for use if an analog volume control is
// available on the capture device. It will require the user to provide
// coupling between the OS mixer controls and AGC through the
// stream_analog_level() functions.
// It consists of an analog gain prescription for the audio device and a
// digital compression stage.
kAdaptiveAnalog,
// Adaptive mode intended for situations in which an analog volume
// control is unavailable. It operates in a similar fashion to the
// adaptive analog mode, but with scaling instead applied in the digital
// domain. As with the analog mode, it additionally uses a digital
// compression stage.
kAdaptiveDigital,
// Fixed mode which enables only the digital compression stage also used
// by the two adaptive modes.
// It is distinguished from the adaptive modes by considering only a
// short time-window of the input signal. It applies a fixed gain
// through most of the input level range, and compresses (gradually
// reduces gain with increasing level) the input signal at higher
// levels. This mode is preferred on embedded devices where the capture
// signal level is predictable, so that a known gain can be applied.
kFixedDigital
};
Mode mode = kAdaptiveAnalog;
// Sets the target peak level (or envelope) of the AGC in dBFs (decibels
// from digital full-scale). The convention is to use positive values. For
// instance, passing in a value of 3 corresponds to -3 dBFs, or a target
// level 3 dB below full-scale. Limited to [0, 31].
int target_level_dbfs = 3;
// Sets the maximum gain the digital compression stage may apply, in dB. A
// higher number corresponds to greater compression, while a value of 0
// will leave the signal uncompressed. Limited to [0, 90].
// For updates after APM setup, use a RuntimeSetting instead.
int compression_gain_db = 9;
// When enabled, the compression stage will hard limit the signal to the
// target level. Otherwise, the signal will be compressed but not limited
// above the target level.
bool enable_limiter = true;
// Enables the analog gain controller functionality.
struct AnalogGainController {
bool enabled = true;
// TODO(bugs.webrtc.org/7494): Deprecated. Stop using and remove.
int startup_min_volume = 0;
// Lowest analog microphone level that will be applied in response to
// clipping.
int clipped_level_min = 70;
// If true, an adaptive digital gain is applied.
bool enable_digital_adaptive = true;
// Amount the microphone level is lowered with every clipping event.
// Limited to (0, 255].
int clipped_level_step = 15;
// Proportion of clipped samples required to declare a clipping event.
// Limited to (0.f, 1.f).
float clipped_ratio_threshold = 0.1f;
// Time in frames to wait after a clipping event before checking again.
// Limited to values higher than 0.
int clipped_wait_frames = 300;
// Enables clipping prediction functionality.
struct ClippingPredictor {
bool enabled = false;
enum Mode {
// Clipping event prediction mode with fixed step estimation.
kClippingEventPrediction,
// Clipped peak estimation mode with adaptive step estimation.
kAdaptiveStepClippingPeakPrediction,
// Clipped peak estimation mode with fixed step estimation.
kFixedStepClippingPeakPrediction,
};
Mode mode = kClippingEventPrediction;
// Number of frames in the sliding analysis window.
int window_length = 5;
// Number of frames in the sliding reference window.
int reference_window_length = 5;
// Reference window delay (unit: number of frames).
int reference_window_delay = 5;
// Clipping prediction threshold (dBFS).
float clipping_threshold = -1.0f;
// Crest factor drop threshold (dB).
float crest_factor_margin = 3.0f;
// If true, the recommended clipped level step is used to modify the
// analog gain. Otherwise, the predictor runs without affecting the
// analog gain.
bool use_predicted_step = true;
} clipping_predictor;
} analog_gain_controller;
} gain_controller1;
// Parameters for AGC2, an Automatic Gain Control (AGC) sub-module which
// replaces the AGC sub-module parametrized by `gain_controller1`.
// AGC2 brings the captured audio signal to the desired level by combining
// three different controllers (namely, input volume controller, adapative
// digital controller and fixed digital controller) and a limiter.
// TODO(bugs.webrtc.org:7494): Name `GainController` when AGC1 removed.
struct RTC_EXPORT GainController2 {
bool operator==(const GainController2& rhs) const;
bool operator!=(const GainController2& rhs) const {
return !(*this == rhs);
}
// AGC2 must be created if and only if `enabled` is true.
bool enabled = false;
// Parameters for the input volume controller, which adjusts the input
// volume applied when the audio is captured (e.g., microphone volume on
// a soundcard, input volume on HAL).
struct InputVolumeController {
bool operator==(const InputVolumeController& rhs) const;
bool operator!=(const InputVolumeController& rhs) const {
return !(*this == rhs);
}
bool enabled = false;
} input_volume_controller;
// Parameters for the adaptive digital controller, which adjusts and
// applies a digital gain after echo cancellation and after noise
// suppression.
struct RTC_EXPORT AdaptiveDigital {
bool operator==(const AdaptiveDigital& rhs) const;
bool operator!=(const AdaptiveDigital& rhs) const {
return !(*this == rhs);
}
bool enabled = false;
float headroom_db = 6.0f;
float max_gain_db = 30.0f;
float initial_gain_db = 8.0f;
float max_gain_change_db_per_second = 3.0f;
float max_output_noise_level_dbfs = -50.0f;
} adaptive_digital;
// Parameters for the fixed digital controller, which applies a fixed
// digital gain after the adaptive digital controller and before the
// limiter.
struct FixedDigital {
// By setting `gain_db` to a value greater than zero, the limiter can be
// turned into a compressor that first applies a fixed gain.
float gain_db = 0.0f;
} fixed_digital;
} gain_controller2;
std::string ToString() const;
};
// Specifies the properties of a setting to be passed to AudioProcessing at
// runtime.
class RuntimeSetting {
public:
enum class Type {
kNotSpecified,
kCapturePreGain,
kCaptureCompressionGain,
kCaptureFixedPostGain,
kPlayoutVolumeChange,
kCustomRenderProcessingRuntimeSetting,
kPlayoutAudioDeviceChange,
kCapturePostGain,
kCaptureOutputUsed
};
// Play-out audio device properties.
struct PlayoutAudioDeviceInfo {
int id; // Identifies the audio device.
int max_volume; // Maximum play-out volume.
};
RuntimeSetting() : type_(Type::kNotSpecified), value_(0.0f) {}
~RuntimeSetting() = default;
static RuntimeSetting CreateCapturePreGain(float gain) {
return {Type::kCapturePreGain, gain};
}
static RuntimeSetting CreateCapturePostGain(float gain) {
return {Type::kCapturePostGain, gain};
}
// Corresponds to Config::GainController1::compression_gain_db, but for
// runtime configuration.
static RuntimeSetting CreateCompressionGainDb(int gain_db) {
RTC_DCHECK_GE(gain_db, 0);
RTC_DCHECK_LE(gain_db, 90);
return {Type::kCaptureCompressionGain, static_cast<float>(gain_db)};
}
// Corresponds to Config::GainController2::fixed_digital::gain_db, but for
// runtime configuration.
static RuntimeSetting CreateCaptureFixedPostGain(float gain_db) {
RTC_DCHECK_GE(gain_db, 0.0f);
RTC_DCHECK_LE(gain_db, 90.0f);
return {Type::kCaptureFixedPostGain, gain_db};
}
// Creates a runtime setting to notify play-out (aka render) audio device
// changes.
static RuntimeSetting CreatePlayoutAudioDeviceChange(
PlayoutAudioDeviceInfo audio_device) {
return {Type::kPlayoutAudioDeviceChange, audio_device};
}
// Creates a runtime setting to notify play-out (aka render) volume changes.
// `volume` is the unnormalized volume, the maximum of which
static RuntimeSetting CreatePlayoutVolumeChange(int volume) {
return {Type::kPlayoutVolumeChange, volume};
}
static RuntimeSetting CreateCustomRenderSetting(float payload) {
return {Type::kCustomRenderProcessingRuntimeSetting, payload};
}
static RuntimeSetting CreateCaptureOutputUsedSetting(
bool capture_output_used) {
return {Type::kCaptureOutputUsed, capture_output_used};
}
Type type() const { return type_; }
// Getters do not return a value but instead modify the argument to protect
// from implicit casting.
void GetFloat(float* value) const {
RTC_DCHECK(value);
*value = value_.float_value;
}
void GetInt(int* value) const {
RTC_DCHECK(value);
*value = value_.int_value;
}
void GetBool(bool* value) const {
RTC_DCHECK(value);
*value = value_.bool_value;
}
void GetPlayoutAudioDeviceInfo(PlayoutAudioDeviceInfo* value) const {
RTC_DCHECK(value);
*value = value_.playout_audio_device_info;
}
private:
RuntimeSetting(Type id, float value) : type_(id), value_(value) {}
RuntimeSetting(Type id, int value) : type_(id), value_(value) {}
RuntimeSetting(Type id, PlayoutAudioDeviceInfo value)
: type_(id), value_(value) {}
Type type_;
union U {
U() {}
U(int value) : int_value(value) {}
U(float value) : float_value(value) {}
U(PlayoutAudioDeviceInfo value) : playout_audio_device_info(value) {}
float float_value;
int int_value;
bool bool_value;
PlayoutAudioDeviceInfo playout_audio_device_info;
} value_;
};
~AudioProcessing() override {}
// Initializes internal states, while retaining all user settings. This
// should be called before beginning to process a new audio stream. However,
// it is not necessary to call before processing the first stream after
// creation.
//
// It is also not necessary to call if the audio parameters (sample
// rate and number of channels) have changed. Passing updated parameters
// directly to `ProcessStream()` and `ProcessReverseStream()` is permissible.
// If the parameters are known at init-time though, they may be provided.
// TODO(webrtc:5298): Change to return void.
virtual int Initialize() = 0;
// The int16 interfaces require:
// - only `NativeRate`s be used
// - that the input, output and reverse rates must match
// - that `processing_config.output_stream()` matches
// `processing_config.input_stream()`.
//
// The float interfaces accept arbitrary rates and support differing input and
// output layouts, but the output must have either one channel or the same
// number of channels as the input.
virtual int Initialize(const ProcessingConfig& processing_config) = 0;
// TODO(peah): This method is a temporary solution used to take control
// over the parameters in the audio processing module and is likely to change.
virtual void ApplyConfig(const Config& config) = 0;
// TODO(ajm): Only intended for internal use. Make private and friend the
// necessary classes?
virtual int proc_sample_rate_hz() const = 0;
virtual int proc_split_sample_rate_hz() const = 0;
virtual size_t num_input_channels() const = 0;
virtual size_t num_proc_channels() const = 0;
virtual size_t num_output_channels() const = 0;
virtual size_t num_reverse_channels() const = 0;
// Set to true when the output of AudioProcessing will be muted or in some
// other way not used. Ideally, the captured audio would still be processed,
// but some components may change behavior based on this information.
// Default false. This method takes a lock. To achieve this in a lock-less
// manner the PostRuntimeSetting can instead be used.
virtual void set_output_will_be_muted(bool muted) = 0;
// Enqueues a runtime setting.
virtual void SetRuntimeSetting(RuntimeSetting setting) = 0;
// Enqueues a runtime setting. Returns a bool indicating whether the
// enqueueing was successfull.
virtual bool PostRuntimeSetting(RuntimeSetting setting) = 0;
// Accepts and produces a ~10 ms frame of interleaved 16 bit integer audio as
// specified in `input_config` and `output_config`. `src` and `dest` may use
// the same memory, if desired.
virtual int ProcessStream(const int16_t* const src,
const StreamConfig& input_config,
const StreamConfig& output_config,
int16_t* const dest) = 0;
// Accepts deinterleaved float audio with the range [-1, 1]. Each element of
// `src` points to a channel buffer, arranged according to `input_stream`. At
// output, the channels will be arranged according to `output_stream` in
// `dest`.
//
// The output must have one channel or as many channels as the input. `src`
// and `dest` may use the same memory, if desired.
virtual int ProcessStream(const float* const* src,
const StreamConfig& input_config,
const StreamConfig& output_config,
float* const* dest) = 0;
// Accepts and produces a ~10 ms frame of interleaved 16 bit integer audio for
// the reverse direction audio stream as specified in `input_config` and
// `output_config`. `src` and `dest` may use the same memory, if desired.
virtual int ProcessReverseStream(const int16_t* const src,
const StreamConfig& input_config,
const StreamConfig& output_config,
int16_t* const dest) = 0;
// Accepts deinterleaved float audio with the range [-1, 1]. Each element of
// `data` points to a channel buffer, arranged according to `reverse_config`.
virtual int ProcessReverseStream(const float* const* src,
const StreamConfig& input_config,
const StreamConfig& output_config,
float* const* dest) = 0;
// Accepts deinterleaved float audio with the range [-1, 1]. Each element
// of `data` points to a channel buffer, arranged according to
// `reverse_config`.
virtual int AnalyzeReverseStream(const float* const* data,
const StreamConfig& reverse_config) = 0;
// Returns the most recently produced ~10 ms of the linear AEC output at a
// rate of 16 kHz. If there is more than one capture channel, a mono
// representation of the input is returned. Returns true/false to indicate
// whether an output returned.
virtual bool GetLinearAecOutput(
rtc::ArrayView<std::array<float, 160>> linear_output) const = 0;
// This must be called prior to ProcessStream() if and only if adaptive analog
// gain control is enabled, to pass the current analog level from the audio
// HAL. Must be within the range [0, 255].
virtual void set_stream_analog_level(int level) = 0;
// When an analog mode is set, this should be called after
// `set_stream_analog_level()` and `ProcessStream()` to obtain the recommended
// new analog level for the audio HAL. It is the user's responsibility to
// apply this level.
virtual int recommended_stream_analog_level() const = 0;
// This must be called if and only if echo processing is enabled.
//
// Sets the `delay` in ms between ProcessReverseStream() receiving a far-end
// frame and ProcessStream() receiving a near-end frame containing the
// corresponding echo. On the client-side this can be expressed as
// delay = (t_render - t_analyze) + (t_process - t_capture)
// where,
// - t_analyze is the time a frame is passed to ProcessReverseStream() and
// t_render is the time the first sample of the same frame is rendered by
// the audio hardware.
// - t_capture is the time the first sample of a frame is captured by the
// audio hardware and t_process is the time the same frame is passed to
// ProcessStream().
virtual int set_stream_delay_ms(int delay) = 0;
virtual int stream_delay_ms() const = 0;
// Call to signal that a key press occurred (true) or did not occur (false)
// with this chunk of audio.
virtual void set_stream_key_pressed(bool key_pressed) = 0;
// Creates and attaches an webrtc::AecDump for recording debugging
// information.
// The `worker_queue` may not be null and must outlive the created
// AecDump instance. |max_log_size_bytes == -1| means the log size
// will be unlimited. `handle` may not be null. The AecDump takes
// responsibility for `handle` and closes it in the destructor. A
// return value of true indicates that the file has been
// sucessfully opened, while a value of false indicates that
// opening the file failed.
virtual bool CreateAndAttachAecDump(absl::string_view file_name,
int64_t max_log_size_bytes,
rtc::TaskQueue* worker_queue) = 0;
virtual bool CreateAndAttachAecDump(FILE* handle,
int64_t max_log_size_bytes,
rtc::TaskQueue* worker_queue) = 0;
// TODO(webrtc:5298) Deprecated variant.
// Attaches provided webrtc::AecDump for recording debugging
// information. Log file and maximum file size logic is supposed to
// be handled by implementing instance of AecDump. Calling this
// method when another AecDump is attached resets the active AecDump
// with a new one. This causes the d-tor of the earlier AecDump to
// be called. The d-tor call may block until all pending logging
// tasks are completed.
virtual void AttachAecDump(std::unique_ptr<AecDump> aec_dump) = 0;
// If no AecDump is attached, this has no effect. If an AecDump is
// attached, it's destructor is called. The d-tor may block until
// all pending logging tasks are completed.
virtual void DetachAecDump() = 0;
// Get audio processing statistics.
virtual AudioProcessingStats GetStatistics() = 0;
// TODO(webrtc:5298) Deprecated variant. The `has_remote_tracks` argument
// should be set if there are active remote tracks (this would usually be true
// during a call). If there are no remote tracks some of the stats will not be
// set by AudioProcessing, because they only make sense if there is at least
// one remote track.
virtual AudioProcessingStats GetStatistics(bool has_remote_tracks) = 0;
// Returns the last applied configuration.
virtual AudioProcessing::Config GetConfig() const = 0;
enum Error {
// Fatal errors.
kNoError = 0,
kUnspecifiedError = -1,
kCreationFailedError = -2,
kUnsupportedComponentError = -3,
kUnsupportedFunctionError = -4,
kNullPointerError = -5,
kBadParameterError = -6,
kBadSampleRateError = -7,
kBadDataLengthError = -8,
kBadNumberChannelsError = -9,
kFileError = -10,
kStreamParameterNotSetError = -11,
kNotEnabledError = -12,
// Warnings are non-fatal.
// This results when a set_stream_ parameter is out of range. Processing
// will continue, but the parameter may have been truncated.
kBadStreamParameterWarning = -13
};
// Native rates supported by the integer interfaces.
enum NativeRate {
kSampleRate8kHz = 8000,
kSampleRate16kHz = 16000,
kSampleRate32kHz = 32000,
kSampleRate48kHz = 48000
};
// TODO(kwiberg): We currently need to support a compiler (Visual C++) that
// complains if we don't explicitly state the size of the array here. Remove
// the size when that's no longer the case.
static constexpr int kNativeSampleRatesHz[4] = {
kSampleRate8kHz, kSampleRate16kHz, kSampleRate32kHz, kSampleRate48kHz};
static constexpr size_t kNumNativeSampleRates =
arraysize(kNativeSampleRatesHz);
static constexpr int kMaxNativeSampleRateHz =
kNativeSampleRatesHz[kNumNativeSampleRates - 1];
// APM processes audio in chunks of about 10 ms. See GetFrameSize() for
// details.
static constexpr int kChunkSizeMs = 10;
// Returns floor(sample_rate_hz/100): the number of samples per channel used
// as input and output to the audio processing module in calls to
// ProcessStream, ProcessReverseStream, AnalyzeReverseStream, and
// GetLinearAecOutput.
//
// This is exactly 10 ms for sample rates divisible by 100. For example:
// - 48000 Hz (480 samples per channel),
// - 44100 Hz (441 samples per channel),
// - 16000 Hz (160 samples per channel).
//
// Sample rates not divisible by 100 are received/produced in frames of
// approximately 10 ms. For example:
// - 22050 Hz (220 samples per channel, or ~9.98 ms per frame),
// - 11025 Hz (110 samples per channel, or ~9.98 ms per frame).
// These nondivisible sample rates yield lower audio quality compared to
// multiples of 100. Internal resampling to 10 ms frames causes a simulated
// clock drift effect which impacts the performance of (for example) echo
// cancellation.
static int GetFrameSize(int sample_rate_hz) { return sample_rate_hz / 100; }
};
class RTC_EXPORT AudioProcessingBuilder {
public:
AudioProcessingBuilder();
AudioProcessingBuilder(const AudioProcessingBuilder&) = delete;
AudioProcessingBuilder& operator=(const AudioProcessingBuilder&) = delete;
~AudioProcessingBuilder();
// Sets the APM configuration.
AudioProcessingBuilder& SetConfig(const AudioProcessing::Config& config) {
config_ = config;
return *this;
}
// Sets the echo controller factory to inject when APM is created.
AudioProcessingBuilder& SetEchoControlFactory(
std::unique_ptr<EchoControlFactory> echo_control_factory) {
echo_control_factory_ = std::move(echo_control_factory);
return *this;
}
// Sets the capture post-processing sub-module to inject when APM is created.
AudioProcessingBuilder& SetCapturePostProcessing(
std::unique_ptr<CustomProcessing> capture_post_processing) {
capture_post_processing_ = std::move(capture_post_processing);
return *this;
}
// Sets the render pre-processing sub-module to inject when APM is created.
AudioProcessingBuilder& SetRenderPreProcessing(
std::unique_ptr<CustomProcessing> render_pre_processing) {
render_pre_processing_ = std::move(render_pre_processing);
return *this;
}
// Sets the echo detector to inject when APM is created.
AudioProcessingBuilder& SetEchoDetector(
rtc::scoped_refptr<EchoDetector> echo_detector) {
echo_detector_ = std::move(echo_detector);
return *this;
}
// Sets the capture analyzer sub-module to inject when APM is created.
AudioProcessingBuilder& SetCaptureAnalyzer(
std::unique_ptr<CustomAudioAnalyzer> capture_analyzer) {
capture_analyzer_ = std::move(capture_analyzer);
return *this;
}
// Creates an APM instance with the specified config or the default one if
// unspecified. Injects the specified components transferring the ownership
// to the newly created APM instance - i.e., except for the config, the
// builder is reset to its initial state.
rtc::scoped_refptr<AudioProcessing> Create();
private:
AudioProcessing::Config config_;
std::unique_ptr<EchoControlFactory> echo_control_factory_;
std::unique_ptr<CustomProcessing> capture_post_processing_;
std::unique_ptr<CustomProcessing> render_pre_processing_;
rtc::scoped_refptr<EchoDetector> echo_detector_;
std::unique_ptr<CustomAudioAnalyzer> capture_analyzer_;
};
class StreamConfig {
public:
// sample_rate_hz: The sampling rate of the stream.
// num_channels: The number of audio channels in the stream.
StreamConfig(int sample_rate_hz = 0, size_t num_channels = 0)
: sample_rate_hz_(sample_rate_hz),
num_channels_(num_channels),
num_frames_(calculate_frames(sample_rate_hz)) {}
void set_sample_rate_hz(int value) {
sample_rate_hz_ = value;
num_frames_ = calculate_frames(value);
}
void set_num_channels(size_t value) { num_channels_ = value; }
int sample_rate_hz() const { return sample_rate_hz_; }
// The number of channels in the stream.
size_t num_channels() const { return num_channels_; }
size_t num_frames() const { return num_frames_; }
size_t num_samples() const { return num_channels_ * num_frames_; }
bool operator==(const StreamConfig& other) const {
return sample_rate_hz_ == other.sample_rate_hz_ &&
num_channels_ == other.num_channels_;
}
bool operator!=(const StreamConfig& other) const { return !(*this == other); }
private:
static size_t calculate_frames(int sample_rate_hz) {
return static_cast<size_t>(AudioProcessing::GetFrameSize(sample_rate_hz));
}
int sample_rate_hz_;
size_t num_channels_;
size_t num_frames_;
};
class ProcessingConfig {
public:
enum StreamName {
kInputStream,
kOutputStream,
kReverseInputStream,
kReverseOutputStream,
kNumStreamNames,
};
const StreamConfig& input_stream() const {
return streams[StreamName::kInputStream];
}
const StreamConfig& output_stream() const {
return streams[StreamName::kOutputStream];
}
const StreamConfig& reverse_input_stream() const {
return streams[StreamName::kReverseInputStream];
}
const StreamConfig& reverse_output_stream() const {
return streams[StreamName::kReverseOutputStream];
}
StreamConfig& input_stream() { return streams[StreamName::kInputStream]; }
StreamConfig& output_stream() { return streams[StreamName::kOutputStream]; }
StreamConfig& reverse_input_stream() {
return streams[StreamName::kReverseInputStream];
}
StreamConfig& reverse_output_stream() {
return streams[StreamName::kReverseOutputStream];
}
bool operator==(const ProcessingConfig& other) const {
for (int i = 0; i < StreamName::kNumStreamNames; ++i) {
if (this->streams[i] != other.streams[i]) {
return false;
}
}
return true;
}
bool operator!=(const ProcessingConfig& other) const {
return !(*this == other);
}
StreamConfig streams[StreamName::kNumStreamNames];
};
// Experimental interface for a custom analysis submodule.
class CustomAudioAnalyzer {
public:
// (Re-) Initializes the submodule.
virtual void Initialize(int sample_rate_hz, int num_channels) = 0;
// Analyzes the given capture or render signal.
virtual void Analyze(const AudioBuffer* audio) = 0;
// Returns a string representation of the module state.
virtual std::string ToString() const = 0;
virtual ~CustomAudioAnalyzer() {}
};
// Interface for a custom processing submodule.
class CustomProcessing {
public:
// (Re-)Initializes the submodule.
virtual void Initialize(int sample_rate_hz, int num_channels) = 0;
// Processes the given capture or render signal.
virtual void Process(AudioBuffer* audio) = 0;
// Returns a string representation of the module state.
virtual std::string ToString() const = 0;
// Handles RuntimeSettings. TODO(webrtc:9262): make pure virtual
// after updating dependencies.
virtual void SetRuntimeSetting(AudioProcessing::RuntimeSetting setting);
virtual ~CustomProcessing() {}
};
// Interface for an echo detector submodule.
class EchoDetector : public rtc::RefCountInterface {
public:
// (Re-)Initializes the submodule.
virtual void Initialize(int capture_sample_rate_hz,
int num_capture_channels,
int render_sample_rate_hz,
int num_render_channels) = 0;
// Analysis (not changing) of the first channel of the render signal.
virtual void AnalyzeRenderAudio(rtc::ArrayView<const float> render_audio) = 0;
// Analysis (not changing) of the capture signal.
virtual void AnalyzeCaptureAudio(
rtc::ArrayView<const float> capture_audio) = 0;
struct Metrics {
absl::optional<double> echo_likelihood;
absl::optional<double> echo_likelihood_recent_max;
};
// Collect current metrics from the echo detector.
virtual Metrics GetMetrics() const = 0;
};
} // namespace webrtc
// This is a transitional header forwarding to the new version in the api/
// folder.
#include "api/audio/audio_processing.h"
#endif // MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_H_

22
webrtc/modules/audio_processing/include/audio_processing_statistics.cc
View File

@ -1,22 +0,0 @@
/*
* Copyright 2017 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/audio_processing/include/audio_processing_statistics.h"
namespace webrtc {
AudioProcessingStats::AudioProcessingStats() = default;
AudioProcessingStats::AudioProcessingStats(const AudioProcessingStats& other) =
default;
AudioProcessingStats::~AudioProcessingStats() = default;
} // namespace webrtc

55
webrtc/modules/audio_processing/include/audio_processing_statistics.h
View File

@ -11,57 +11,8 @@
#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_STATISTICS_H_
#define MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_STATISTICS_H_
#include <stdint.h>
#include "absl/types/optional.h"
#include "rtc_base/system/rtc_export.h"
namespace webrtc {
// This version of the stats uses Optionals, it will replace the regular
// AudioProcessingStatistics struct.
struct RTC_EXPORT AudioProcessingStats {
AudioProcessingStats();
AudioProcessingStats(const AudioProcessingStats& other);
~AudioProcessingStats();
// Deprecated.
// TODO(bugs.webrtc.org/11226): Remove.
// True if voice is detected in the last capture frame, after processing.
// It is conservative in flagging audio as speech, with low likelihood of
// incorrectly flagging a frame as voice.
// Only reported if voice detection is enabled in AudioProcessing::Config.
absl::optional<bool> voice_detected;
// AEC Statistics.
// ERL = 10log_10(P_far / P_echo)
absl::optional<double> echo_return_loss;
// ERLE = 10log_10(P_echo / P_out)
absl::optional<double> echo_return_loss_enhancement;
// Fraction of time that the AEC linear filter is divergent, in a 1-second
// non-overlapped aggregation window.
absl::optional<double> divergent_filter_fraction;
// The delay metrics consists of the delay median and standard deviation. It
// also consists of the fraction of delay estimates that can make the echo
// cancellation perform poorly. The values are aggregated until the first
// call to `GetStatistics()` and afterwards aggregated and updated every
// second. Note that if there are several clients pulling metrics from
// `GetStatistics()` during a session the first call from any of them will
// change to one second aggregation window for all.
absl::optional<int32_t> delay_median_ms;
absl::optional<int32_t> delay_standard_deviation_ms;
// Residual echo detector likelihood.
absl::optional<double> residual_echo_likelihood;
// Maximum residual echo likelihood from the last time period.
absl::optional<double> residual_echo_likelihood_recent_max;
// The instantaneous delay estimate produced in the AEC. The unit is in
// milliseconds and the value is the instantaneous value at the time of the
// call to `GetStatistics()`.
absl::optional<int32_t> delay_ms;
};
} // namespace webrtc
// This is a transitional header forwarding to the new version in the api/
// folder.
#include "api/audio/audio_processing_statistics.h"
#endif // MODULES_AUDIO_PROCESSING_INCLUDE_AUDIO_PROCESSING_STATISTICS_H_

23
webrtc/modules/audio_processing/include/config.cc
View File

@ -1,23 +0,0 @@
/*
* Copyright (c) 2016 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/audio_processing/include/config.h"
namespace webrtc {
Config::Config() {}
Config::~Config() {
for (OptionMap::iterator it = options_.begin(); it != options_.end(); ++it) {
delete it->second;
}
}
} // namespace webrtc

131
webrtc/modules/audio_processing/include/config.h
View File

@ -1,131 +0,0 @@
/*
* Copyright (c) 2013 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#ifndef MODULES_AUDIO_PROCESSING_INCLUDE_CONFIG_H_
#define MODULES_AUDIO_PROCESSING_INCLUDE_CONFIG_H_
#include <map>
#include "rtc_base/system/rtc_export.h"
namespace webrtc {
// Only add new values to the end of the enumeration and never remove (only
// deprecate) to maintain binary compatibility.
enum class ConfigOptionID {
kMyExperimentForTest,
kAlgo1CostFunctionForTest,
kTemporalLayersFactory, // Deprecated
kNetEqCapacityConfig, // Deprecated
kNetEqFastAccelerate, // Deprecated
kVoicePacing, // Deprecated
kExtendedFilter, // Deprecated
kDelayAgnostic, // Deprecated
kExperimentalAgc,
kExperimentalNs,
kBeamforming, // Deprecated
kIntelligibility, // Deprecated
kEchoCanceller3, // Deprecated
kAecRefinedAdaptiveFilter, // Deprecated
kLevelControl // Deprecated
};
// Class Config is designed to ease passing a set of options across webrtc code.
// Options are identified by typename in order to avoid incorrect casts.
//
// Usage:
// * declaring an option:
// struct Algo1_CostFunction {
// virtual float cost(int x) const { return x; }
// virtual ~Algo1_CostFunction() {}
// };
//
// * accessing an option:
// config.Get<Algo1_CostFunction>().cost(value);
//
// * setting an option:
// struct SqrCost : Algo1_CostFunction {
// virtual float cost(int x) const { return x*x; }
// };
// config.Set<Algo1_CostFunction>(new SqrCost());
//
// Note: This class is thread-compatible (like STL containers).
class RTC_EXPORT Config {
public:
// Returns the option if set or a default constructed one.
// Callers that access options too often are encouraged to cache the result.
// Returned references are owned by this.
//
// Requires std::is_default_constructible<T>
template <typename T>
const T& Get() const;
// Set the option, deleting any previous instance of the same.
// This instance gets ownership of the newly set value.
template <typename T>
void Set(T* value);
Config();
~Config();
private:
struct BaseOption {
virtual ~BaseOption() {}
};
template <typename T>
struct Option : BaseOption {
explicit Option(T* v) : value(v) {}
~Option() { delete value; }
T* value;
};
template <typename T>
static ConfigOptionID identifier() {
return T::identifier;
}
// Used to instantiate a default constructed object that doesn't needs to be
// owned. This allows Get<T> to be implemented without requiring explicitly
// locks.
template <typename T>
static const T& default_value() {
static const T* const def = new T();
return *def;
}
typedef std::map<ConfigOptionID, BaseOption*> OptionMap;
OptionMap options_;
Config(const Config&);
void operator=(const Config&);
};
template <typename T>
const T& Config::Get() const {
OptionMap::const_iterator it = options_.find(identifier<T>());
if (it != options_.end()) {
const T* t = static_cast<Option<T>*>(it->second)->value;
if (t) {
return *t;
}
}
return default_value<T>();
}
template <typename T>
void Config::Set(T* value) {
BaseOption*& it = options_[identifier<T>()];
delete it;
it = new Option<T>(value);
}
} // namespace webrtc
#endif // MODULES_AUDIO_PROCESSING_INCLUDE_CONFIG_H_

2
webrtc/modules/audio_processing/logging/apm_data_dumper.cc
View File

@ -61,7 +61,7 @@ ApmDataDumper::~ApmDataDumper() = default;
#if WEBRTC_APM_DEBUG_DUMP == 1
bool ApmDataDumper::recording_activated_ = false;
absl::optional<int> ApmDataDumper::dump_set_to_use_;
std::optional<int> ApmDataDumper::dump_set_to_use_;
char ApmDataDumper::output_dir_[] = "";
FILE* ApmDataDumper::GetRawFile(absl::string_view name) {

5
webrtc/modules/audio_processing/logging/apm_data_dumper.h
View File

@ -20,8 +20,9 @@
#include <unordered_map>
#endif
#include <optional>
#include "absl/strings/string_view.h"
#include "absl/types/optional.h"
#include "api/array_view.h"
#if WEBRTC_APM_DEBUG_DUMP == 1
#include "common_audio/wav_file.h"
@ -391,7 +392,7 @@ class ApmDataDumper {
private:
#if WEBRTC_APM_DEBUG_DUMP == 1
static bool recording_activated_;
static absl::optional<int> dump_set_to_use_;
static std::optional<int> dump_set_to_use_;
static constexpr size_t kOutputDirMaxLength = 1024;
static char output_dir_[kOutputDirMaxLength];
const int instance_index_;

14
webrtc/modules/audio_processing/meson.build
View File

@ -114,9 +114,6 @@ webrtc_audio_processing_sources = [
'high_pass_filter.cc',
'include/aec_dump.cc',
'include/audio_frame_proxies.cc',
'include/audio_processing.cc',
'include/audio_processing_statistics.cc',
'include/config.cc',
'logging/apm_data_dumper.cc',
'ns/fast_math.cc',
'ns/histograms.cc',
@ -131,18 +128,10 @@ webrtc_audio_processing_sources = [
'ns/speech_probability_estimator.cc',
'ns/suppression_params.cc',
'ns/wiener_filter.cc',
'optionally_built_submodule_creators.cc',
'residual_echo_detector.cc',
'rms_level.cc',
'splitting_filter.cc',
'three_band_filter_bank.cc',
'transient/file_utils.cc',
'transient/moving_moments.cc',
'transient/transient_detector.cc',
'transient/transient_suppressor_impl.cc',
'transient/voice_probability_delay_unit.cc',
'transient/wpd_node.cc',
'transient/wpd_tree.cc',
'utility/cascaded_biquad_filter.cc',
'utility/delay_estimator.cc',
'utility/delay_estimator_wrapper.cc',
@ -160,7 +149,6 @@ webrtc_audio_processing_sources = [
webrtc_audio_processing_include_headers = [
'include/audio_processing.h',
'include/audio_processing_statistics.h',
'include/config.h',
]
extra_libs = []
@ -204,9 +192,9 @@ libwebrtc_audio_processing = library(apm_project_name,
dependencies: [
base_dep,
api_dep,
isac_vad_dep,
system_wrappers_dep,
common_audio_dep,
isac_vad_dep,
pffft_dep,
rnnoise_dep,
] + common_deps,

2
webrtc/modules/audio_processing/ns/BUILD.gn
View File

@ -64,7 +64,6 @@ rtc_static_library("ns") {
"../../../system_wrappers:metrics",
"../utility:cascaded_biquad_filter",
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
}
if (rtc_include_tests) {
@ -89,7 +88,6 @@ if (rtc_include_tests) {
"../../../test:test_support",
"../utility:cascaded_biquad_filter",
]
absl_deps = [ "//third_party/abseil-cpp/absl/types:optional" ]
defines = []

36
webrtc/modules/audio_processing/optionally_built_submodule_creators.cc
View File

@ -1,36 +0,0 @@
/*
* Copyright (c) 2020 The WebRTC project authors. All Rights Reserved.
*
* Use of this source code is governed by a BSD-style license
* that can be found in the LICENSE file in the root of the source
* tree. An additional intellectual property rights grant can be found
* in the file PATENTS. All contributing project authors may
* be found in the AUTHORS file in the root of the source tree.
*/
#include "modules/audio_processing/optionally_built_submodule_creators.h"
#include <memory>
#include "modules/audio_processing/transient/transient_suppressor_impl.h"
namespace webrtc {
std::unique_ptr<TransientSuppressor> CreateTransientSuppressor(
const ApmSubmoduleCreationOverrides& overrides,
TransientSuppressor::VadMode vad_mode,
int sample_rate_hz,
int detection_rate_hz,
int num_channels) {
#ifdef WEBRTC_EXCLUDE_TRANSIENT_SUPPRESSOR
return nullptr;
#else
if (overrides.transient_suppression) {
return nullptr;
}
return std::make_unique<TransientSuppressorImpl>(
vad_mode, sample_rate_hz, detection_rate_hz, num_channels);
#endif
}
} // namespace webrtc

Some files were not shown because too many files have changed in this diff Show More